<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
  <meta charset="utf-8" />
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
  <title>graph4nlp.data &mdash; Graph4NLP v0.4.1 documentation</title><link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
    <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <!--[if lt IE 9]>
    <script src="../_static/js/html5shiv.min.js"></script>
  <![endif]-->
  <script id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
        <script src="../_static/jquery.js"></script>
        <script src="../_static/underscore.js"></script>
        <script src="../_static/doctools.js"></script>
        <script src="../_static/language_data.js"></script>
        <script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
    <script src="../_static/js/theme.js"></script>
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="next" title="graph4nlp.datasets" href="datasets.html" />
    <link rel="prev" title="Chapter 7. Evaluations and Loss components" href="../guide/evaluation.html" /> 
</head>

<body class="wy-body-for-nav"> 
  <div class="wy-grid-for-nav">
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
            <a href="../index.html" class="icon icon-home"> Graph4NLP
          </a>
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>
        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
              <p class="caption"><span class="caption-text">Get Started</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../welcome/installation.html">Install Graph4NLP</a></li>
</ul>
<p class="caption"><span class="caption-text">User Guide</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../guide/graphdata.html">Chapter 1. Graph Data</a></li>
<li class="toctree-l1"><a class="reference internal" href="../guide/dataset.html">Chapter 2. Dataset</a></li>
<li class="toctree-l1"><a class="reference internal" href="../guide/construction.html">Chapter 3. Graph Construction</a></li>
<li class="toctree-l1"><a class="reference internal" href="../guide/gnn.html">Chapter 4. Graph Encoder</a></li>
<li class="toctree-l1"><a class="reference internal" href="../guide/decoding.html">Chapter 5. Decoder</a></li>
<li class="toctree-l1"><a class="reference internal" href="../guide/classification.html">Chapter 6. Classification</a></li>
<li class="toctree-l1"><a class="reference internal" href="../guide/evaluation.html">Chapter 7. Evaluations and Loss components</a></li>
</ul>
<p class="caption"><span class="caption-text">Module API references</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="current reference internal" href="#">graph4nlp.data</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#graph-representation">Graph Representation</a></li>
<li class="toctree-l2"><a class="reference internal" href="#base-dataset-class">Base Dataset Class</a></li>
<li class="toctree-l2"><a class="reference internal" href="#task-level-dataset-base-class">Task Level Dataset Base Class</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="datasets.html">graph4nlp.datasets</a></li>
<li class="toctree-l1"><a class="reference internal" href="graph_construction.html">graph4nlp.graph_construction</a></li>
<li class="toctree-l1"><a class="reference internal" href="graph_embedding.html">graph4nlp.graph_embedding</a></li>
<li class="toctree-l1"><a class="reference internal" href="prediction.html">graph4nlp.prediction</a></li>
<li class="toctree-l1"><a class="reference internal" href="loss.html">graph4nlp.loss</a></li>
<li class="toctree-l1"><a class="reference internal" href="evaluation.html">graph4nlp.evaluation</a></li>
</ul>
<p class="caption"><span class="caption-text">Tutorials</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../tutorial/text_classification.html">Text Classification Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial/semantic_parsing.html">Semantic Parsing Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial/math_word_problem.html">Math Word Problem Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial/knowledge_graph_completion.html">Knowledge Graph Completion Tutorial</a></li>
</ul>

        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../index.html">Graph4NLP</a>
      </nav>

      <div class="wy-nav-content">
        <div class="rst-content">
          <div role="navigation" aria-label="Page navigation">
  <ul class="wy-breadcrumbs">
      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
      <li>graph4nlp.data</li>
      <li class="wy-breadcrumbs-aside">
            <a href="../_sources/modules/data.rst.txt" rel="nofollow"> View page source</a>
      </li>
  </ul>
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
             
  <div class="section" id="module-graph4nlp.data.data">
<span id="graph4nlp-data"></span><h1>graph4nlp.data<a class="headerlink" href="#module-graph4nlp.data.data" title="Permalink to this headline">¶</a></h1>
<p>The Graph4NLP library uses the class <a class="reference internal" href="#graph4nlp.data.data.GraphData" title="graph4nlp.data.data.GraphData"><code class="xref py py-class docutils literal notranslate"><span class="pre">GraphData</span></code></a> as the representation
for structured data (graphs). <a class="reference internal" href="#graph4nlp.data.data.GraphData" title="graph4nlp.data.data.GraphData"><code class="xref py py-class docutils literal notranslate"><span class="pre">GraphData</span></code></a> supports basic operations to
the graph, including adding nodes and edges. <a class="reference internal" href="#graph4nlp.data.data.GraphData" title="graph4nlp.data.data.GraphData"><code class="xref py py-class docutils literal notranslate"><span class="pre">GraphData</span></code></a> also supports
adding features which are in tensor form, and attributes which are of arbitrary
form to the correspondingnodes or edges. Batching operations is also supported
by <a class="reference internal" href="#graph4nlp.data.data.GraphData" title="graph4nlp.data.data.GraphData"><code class="xref py py-class docutils literal notranslate"><span class="pre">GraphData</span></code></a>.</p>
<div class="section" id="graph-representation">
<h2>Graph Representation<a class="headerlink" href="#graph-representation" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="graph4nlp.data.data.GraphData">
<em class="property">class </em><code class="sig-prename descclassname">graph4nlp.data.data.</code><code class="sig-name descname">GraphData</code><span class="sig-paren">(</span><em class="sig-param">src=None</em>, <em class="sig-param">device: str = None</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.data.GraphData" title="Permalink to this definition">¶</a></dt>
<dd><p>Represent a single graph with additional attributes.</p>
<dl class="field-list simple">
<dt class="field-odd">Attributes</dt>
<dd class="field-odd"><dl class="simple">
<dt><a class="reference internal" href="#graph4nlp.data.data.GraphData.batch_edge_features" title="graph4nlp.data.data.GraphData.batch_edge_features"><code class="xref py py-obj docutils literal notranslate"><span class="pre">batch_edge_features</span></code></a></dt><dd><p>Edge version of self.batch_node_features</p>
</dd>
<dt><a class="reference internal" href="#graph4nlp.data.data.GraphData.batch_node_features" title="graph4nlp.data.data.GraphData.batch_node_features"><code class="xref py py-obj docutils literal notranslate"><span class="pre">batch_node_features</span></code></a></dt><dd><p>Get a view of the batched(padded) version of the node features.</p>
</dd>
<dt><a class="reference internal" href="#graph4nlp.data.data.GraphData.edge_attributes" title="graph4nlp.data.data.GraphData.edge_attributes"><code class="xref py py-obj docutils literal notranslate"><span class="pre">edge_attributes</span></code></a></dt><dd><p>Get the edge attributes in a list.</p>
</dd>
<dt><a class="reference internal" href="#graph4nlp.data.data.GraphData.edge_features" title="graph4nlp.data.data.GraphData.edge_features"><code class="xref py py-obj docutils literal notranslate"><span class="pre">edge_features</span></code></a></dt><dd><p>Get all the edge features in a dictionary.</p>
</dd>
<dt><a class="reference internal" href="#graph4nlp.data.data.GraphData.edges" title="graph4nlp.data.data.GraphData.edges"><code class="xref py py-obj docutils literal notranslate"><span class="pre">edges</span></code></a></dt><dd><p>Return an edge view of the edges and the corresponding data</p>
</dd>
<dt><a class="reference internal" href="#graph4nlp.data.data.GraphData.node_attributes" title="graph4nlp.data.data.GraphData.node_attributes"><code class="xref py py-obj docutils literal notranslate"><span class="pre">node_attributes</span></code></a></dt><dd><p>Access node attribute dictionary</p>
</dd>
<dt><a class="reference internal" href="#graph4nlp.data.data.GraphData.node_features" title="graph4nlp.data.data.GraphData.node_features"><code class="xref py py-obj docutils literal notranslate"><span class="pre">node_features</span></code></a></dt><dd><p>Access and modify node feature vectors (tensor).</p>
</dd>
<dt><a class="reference internal" href="#graph4nlp.data.data.GraphData.nodes" title="graph4nlp.data.data.GraphData.nodes"><code class="xref py py-obj docutils literal notranslate"><span class="pre">nodes</span></code></a></dt><dd><p>Return a node view through which the user can access the features and attributes.</p>
</dd>
<dt><strong>split_edge_features</strong></dt><dd></dd>
<dt><strong>split_node_features</strong></dt><dd></dd>
</dl>
</dd>
</dl>
<p class="rubric">Methods</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.add_edge" title="graph4nlp.data.data.GraphData.add_edge"><code class="xref py py-obj docutils literal notranslate"><span class="pre">add_edge</span></code></a>(src, tgt)</p></td>
<td><p>Add one edge to the graph.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.add_edges" title="graph4nlp.data.data.GraphData.add_edges"><code class="xref py py-obj docutils literal notranslate"><span class="pre">add_edges</span></code></a>(src, tgt)</p></td>
<td><p>Add a bunch of edges to the graph.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.add_nodes" title="graph4nlp.data.data.GraphData.add_nodes"><code class="xref py py-obj docutils literal notranslate"><span class="pre">add_nodes</span></code></a>(node_num)</p></td>
<td><p>Add a number of nodes to the graph.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.adj_matrix" title="graph4nlp.data.data.GraphData.adj_matrix"><code class="xref py py-obj docutils literal notranslate"><span class="pre">adj_matrix</span></code></a>([batch_view, post_processing_fn])</p></td>
<td><p>Returns the adjacency matrix of the graph.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.copy_batch_info" title="graph4nlp.data.data.GraphData.copy_batch_info"><code class="xref py py-obj docutils literal notranslate"><span class="pre">copy_batch_info</span></code></a>(batch)</p></td>
<td><p>Copy all the information related to the batching.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.edge_ids" title="graph4nlp.data.data.GraphData.edge_ids"><code class="xref py py-obj docutils literal notranslate"><span class="pre">edge_ids</span></code></a>(src, tgt)</p></td>
<td><p>Convert the given endpoints to edge indices.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.from_dense_adj" title="graph4nlp.data.data.GraphData.from_dense_adj"><code class="xref py py-obj docutils literal notranslate"><span class="pre">from_dense_adj</span></code></a>(adj)</p></td>
<td><p>Construct a graph from a dense (2-D NxN) adjacency matrix with the edge weights represented by the value of the matrix entries.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.from_dgl" title="graph4nlp.data.data.GraphData.from_dgl"><code class="xref py py-obj docutils literal notranslate"><span class="pre">from_dgl</span></code></a>(dgl_g)</p></td>
<td><p>Build the graph from dgl.DGLGraph</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.from_graphdata" title="graph4nlp.data.data.GraphData.from_graphdata"><code class="xref py py-obj docutils literal notranslate"><span class="pre">from_graphdata</span></code></a>(src)</p></td>
<td><p>Build a clone from a source GraphData</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.from_scipy_sparse_matrix" title="graph4nlp.data.data.GraphData.from_scipy_sparse_matrix"><code class="xref py py-obj docutils literal notranslate"><span class="pre">from_scipy_sparse_matrix</span></code></a>(adj)</p></td>
<td><p>Construct a graph from a sparse adjacency matrix with the edge weights represented by the value of the matrix entries.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.get_all_edges" title="graph4nlp.data.data.GraphData.get_all_edges"><code class="xref py py-obj docutils literal notranslate"><span class="pre">get_all_edges</span></code></a>()</p></td>
<td><p>Get all the edges in the graph</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.get_edge_feature" title="graph4nlp.data.data.GraphData.get_edge_feature"><code class="xref py py-obj docutils literal notranslate"><span class="pre">get_edge_feature</span></code></a>(edges)</p></td>
<td><p>Get the feature of the given edges.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.get_edge_feature_names" title="graph4nlp.data.data.GraphData.get_edge_feature_names"><code class="xref py py-obj docutils literal notranslate"><span class="pre">get_edge_feature_names</span></code></a>()</p></td>
<td><p>Get all the names of edge features</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.get_edge_num" title="graph4nlp.data.data.GraphData.get_edge_num"><code class="xref py py-obj docutils literal notranslate"><span class="pre">get_edge_num</span></code></a>()</p></td>
<td><p>Get the number of edges in the graph</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.get_node_attrs" title="graph4nlp.data.data.GraphData.get_node_attrs"><code class="xref py py-obj docutils literal notranslate"><span class="pre">get_node_attrs</span></code></a>(nodes)</p></td>
<td><p>Get the attributes of the given <cite>nodes</cite>.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.get_node_features" title="graph4nlp.data.data.GraphData.get_node_features"><code class="xref py py-obj docutils literal notranslate"><span class="pre">get_node_features</span></code></a>(nodes)</p></td>
<td><p>Get the node feature dictionary of the <cite>nodes</cite></p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.get_node_num" title="graph4nlp.data.data.GraphData.get_node_num"><code class="xref py py-obj docutils literal notranslate"><span class="pre">get_node_num</span></code></a>()</p></td>
<td><p>Get the number of nodes in the graph.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.node_feature_names" title="graph4nlp.data.data.GraphData.node_feature_names"><code class="xref py py-obj docutils literal notranslate"><span class="pre">node_feature_names</span></code></a>()</p></td>
<td><p>Get the names of node features.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.remove_all_edges" title="graph4nlp.data.data.GraphData.remove_all_edges"><code class="xref py py-obj docutils literal notranslate"><span class="pre">remove_all_edges</span></code></a>()</p></td>
<td><p>Remove all the edges and the corresponding features and attributes in GraphData.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.set_edge_feature" title="graph4nlp.data.data.GraphData.set_edge_feature"><code class="xref py py-obj docutils literal notranslate"><span class="pre">set_edge_feature</span></code></a>(edges, new_data)</p></td>
<td><p>Set edge feature</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.set_node_features" title="graph4nlp.data.data.GraphData.set_node_features"><code class="xref py py-obj docutils literal notranslate"><span class="pre">set_node_features</span></code></a>(nodes, new_data)</p></td>
<td><p>Set the features of the <cite>nodes</cite> with the given <cite>new_data`</cite>.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.sparse_adj" title="graph4nlp.data.data.GraphData.sparse_adj"><code class="xref py py-obj docutils literal notranslate"><span class="pre">sparse_adj</span></code></a>([batch_view])</p></td>
<td><p>Return the scipy.sparse.coo_matrix form of the adjacency matrix :param batch_view: Whether to return the split view of the adjacency matrix.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.split_features" title="graph4nlp.data.data.GraphData.split_features"><code class="xref py py-obj docutils literal notranslate"><span class="pre">split_features</span></code></a>(input_tensor[, type])</p></td>
<td><p>Convert a tensor from [N, <a href="#id1"><span class="problematic" id="id2">*</span></a>] to [B, N_max, <a href="#id3"><span class="problematic" id="id4">*</span></a>] with zero padding according to the batch information stored in the graph.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.to" title="graph4nlp.data.data.GraphData.to"><code class="xref py py-obj docutils literal notranslate"><span class="pre">to</span></code></a>(device)</p></td>
<td><p>Move the GraphData object to different devices(cpu, gpu, etc.).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.data.GraphData.to_dgl" title="graph4nlp.data.data.GraphData.to_dgl"><code class="xref py py-obj docutils literal notranslate"><span class="pre">to_dgl</span></code></a>()</p></td>
<td><p>Convert to dgl.DGLGraph Note that there will be some information loss when calling this function, e.g.</p></td>
</tr>
</tbody>
</table>
<dl class="method">
<dt id="graph4nlp.data.data.GraphData.add_edge">
<code class="sig-name descname">add_edge</code><span class="sig-paren">(</span><em class="sig-param">src: int</em>, <em class="sig-param">tgt: int</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.data.GraphData.add_edge" title="Permalink to this definition">¶</a></dt>
<dd><p>Add one edge to the graph.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>src</strong><span class="classifier">int</span></dt><dd><p>Source node index</p>
</dd>
<dt><strong>tgt</strong><span class="classifier">int</span></dt><dd><p>Target node index</p>
</dd>
</dl>
</dd>
<dt class="field-even">Raises</dt>
<dd class="field-even"><dl class="simple">
<dt>ValueError</dt><dd><p>If one of the endpoints of the edge doesn’t exist in the graph.</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.add_edges">
<code class="sig-name descname">add_edges</code><span class="sig-paren">(</span><em class="sig-param">src: Union[int, List[int]], tgt: Union[int, List[int]]</em><span class="sig-paren">)</span> &#x2192; None<a class="headerlink" href="#graph4nlp.data.data.GraphData.add_edges" title="Permalink to this definition">¶</a></dt>
<dd><p>Add a bunch of edges to the graph.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>src</strong><span class="classifier">int or list</span></dt><dd><p>Source node indices</p>
</dd>
<dt><strong>tgt</strong><span class="classifier">int or list</span></dt><dd><p>Target node indices</p>
</dd>
</dl>
</dd>
<dt class="field-even">Raises</dt>
<dd class="field-even"><dl class="simple">
<dt>ValueError</dt><dd><p>If the lengths of <cite>src</cite> and <cite>tgt</cite> don’t match or one of the list contains no element.</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.add_nodes">
<code class="sig-name descname">add_nodes</code><span class="sig-paren">(</span><em class="sig-param">node_num: int</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.data.GraphData.add_nodes" title="Permalink to this definition">¶</a></dt>
<dd><p>Add a number of nodes to the graph.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>node_num</strong> (<em>int</em>) – The number of nodes to be added</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.adj_matrix">
<code class="sig-name descname">adj_matrix</code><span class="sig-paren">(</span><em class="sig-param">batch_view: bool = False</em>, <em class="sig-param">post_processing_fn: Callable = None</em><span class="sig-paren">)</span> &#x2192; torch.Tensor<a class="headerlink" href="#graph4nlp.data.data.GraphData.adj_matrix" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns the adjacency matrix of the graph. Returns a 2D tensor if
it is a single graph and a 3D tensor if it is a batched graph, with
the matrices padded with 0 (B x N x N)</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>batch_view</strong><span class="classifier">bool</span></dt><dd><p>Whether to return a batched view of the adjacency matrix(3D, True) or not (2D).</p>
</dd>
<dt><strong>post_processing_fn</strong><span class="classifier">function</span></dt><dd><p>A callback function which takes a binary adjacency matrix (2D)
and do some post-processing on it.
The return of this function should also be N x N</p>
</dd>
</dl>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><dl class="simple">
<dt>torch.Tensor:</dt><dd><p>The adjacency matrix (N x N if batch_view=False and B x N x N if batch_view=True).</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.batch_edge_features">
<em class="property">property </em><code class="sig-name descname">batch_edge_features</code><a class="headerlink" href="#graph4nlp.data.data.GraphData.batch_edge_features" title="Permalink to this definition">¶</a></dt>
<dd><p>Edge version of self.batch_node_features</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><dl class="simple">
<dt>BatchEdgeFeatView</dt><dd></dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.batch_node_features">
<em class="property">property </em><code class="sig-name descname">batch_node_features</code><a class="headerlink" href="#graph4nlp.data.data.GraphData.batch_node_features" title="Permalink to this definition">¶</a></dt>
<dd><p>(B, N, D)</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><dl class="simple">
<dt>BatchNodeFeatView</dt><dd></dd>
</dl>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Type</dt>
<dd class="field-odd"><p>Get a view of the batched(padded) version of the node features. Shape</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.copy_batch_info">
<code class="sig-name descname">copy_batch_info</code><span class="sig-paren">(</span><em class="sig-param">batch: Any</em><span class="sig-paren">)</span> &#x2192; None<a class="headerlink" href="#graph4nlp.data.data.GraphData.copy_batch_info" title="Permalink to this definition">¶</a></dt>
<dd><p>Copy all the information related to the batching.
:param batch: The source batch from which the information comes.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><dl class="simple">
<dt>None</dt><dd></dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.edge_attributes">
<em class="property">property </em><code class="sig-name descname">edge_attributes</code><a class="headerlink" href="#graph4nlp.data.data.GraphData.edge_attributes" title="Permalink to this definition">¶</a></dt>
<dd><p>Get the edge attributes in a list.
:returns: A list of dictionaries. Each dictionary represents all</p>
<blockquote>
<div><p>the attributes on the corresponding edge.</p>
</div></blockquote>
<dl class="field-list simple">
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p>list</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.edge_features">
<em class="property">property </em><code class="sig-name descname">edge_features</code><a class="headerlink" href="#graph4nlp.data.data.GraphData.edge_features" title="Permalink to this definition">¶</a></dt>
<dd><p>Get all the edge features in a dictionary.
:returns: Edge features with the keys being the feature names and values</p>
<blockquote>
<div><p>be the corresponding tensors.</p>
</div></blockquote>
<dl class="field-list simple">
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p>dict</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.edge_ids">
<code class="sig-name descname">edge_ids</code><span class="sig-paren">(</span><em class="sig-param">src: Union[int, List[int]], tgt: int</em><span class="sig-paren">)</span> &#x2192; List[Any]<a class="headerlink" href="#graph4nlp.data.data.GraphData.edge_ids" title="Permalink to this definition">¶</a></dt>
<dd><p>Convert the given endpoints to edge indices.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>src</strong><span class="classifier">int or list</span></dt><dd><p>The index of source node(s).</p>
</dd>
<dt><strong>tgt</strong><span class="classifier">int or list</span></dt><dd><p>The index of target node(s).</p>
</dd>
</dl>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><dl class="simple">
<dt>list</dt><dd><p>The index of corresponding edges.</p>
</dd>
</dl>
</dd>
<dt class="field-odd">Raises</dt>
<dd class="field-odd"><dl class="simple">
<dt>TypeError</dt><dd><p>If the parameters are of wrong types.</p>
</dd>
<dt>EdgeNotFoundException</dt><dd><p>If the edge is not in the graph.</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.edges">
<em class="property">property </em><code class="sig-name descname">edges</code><a class="headerlink" href="#graph4nlp.data.data.GraphData.edges" title="Permalink to this definition">¶</a></dt>
<dd><p>Return an edge view of the edges and the corresponding data</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>edges</strong><span class="classifier">EdgeView</span></dt><dd></dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.from_dense_adj">
<code class="sig-name descname">from_dense_adj</code><span class="sig-paren">(</span><em class="sig-param">adj: torch.Tensor</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.data.GraphData.from_dense_adj" title="Permalink to this definition">¶</a></dt>
<dd><p>Construct a graph from a dense (2-D NxN) adjacency matrix with
the edge weights represented by the value of the matrix entries.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>adj</strong><span class="classifier">torch.Tensor</span></dt><dd><p>The tensor representing the adjacency matrix.</p>
</dd>
</dl>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><dl class="simple">
<dt>self</dt><dd></dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.from_dgl">
<code class="sig-name descname">from_dgl</code><span class="sig-paren">(</span><em class="sig-param">dgl_g: dgl.heterograph.DGLHeteroGraph</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.data.GraphData.from_dgl" title="Permalink to this definition">¶</a></dt>
<dd><p>Build the graph from dgl.DGLGraph</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>dgl_g</strong><span class="classifier">dgl.DGLGraph</span></dt><dd><p>The source graph</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.from_graphdata">
<code class="sig-name descname">from_graphdata</code><span class="sig-paren">(</span><em class="sig-param">src: Any</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.data.GraphData.from_graphdata" title="Permalink to this definition">¶</a></dt>
<dd><p>Build a clone from a source GraphData</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.from_scipy_sparse_matrix">
<code class="sig-name descname">from_scipy_sparse_matrix</code><span class="sig-paren">(</span><em class="sig-param">adj: scipy.sparse.coo.coo_matrix</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.data.GraphData.from_scipy_sparse_matrix" title="Permalink to this definition">¶</a></dt>
<dd><p>Construct a graph from a sparse adjacency matrix with the edge weights
represented by the value of the matrix entries.
:param adj: The object representing the sparse adjacency matrix.
:type adj: scipy.sparse.coo_matrix</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><dl class="simple">
<dt>self</dt><dd></dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.get_all_edges">
<code class="sig-name descname">get_all_edges</code><span class="sig-paren">(</span><span class="sig-paren">)</span> &#x2192; List[Tuple[int, int]]<a class="headerlink" href="#graph4nlp.data.data.GraphData.get_all_edges" title="Permalink to this definition">¶</a></dt>
<dd><p>Get all the edges in the graph</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>edges</strong><span class="classifier">list</span></dt><dd><p>List of edges. Each edge is in the shape of the endpoint tuple (src, dst).</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.get_edge_feature">
<code class="sig-name descname">get_edge_feature</code><span class="sig-paren">(</span><em class="sig-param">edges: List[int]</em><span class="sig-paren">)</span> &#x2192; Dict[str, torch.Tensor]<a class="headerlink" href="#graph4nlp.data.data.GraphData.get_edge_feature" title="Permalink to this definition">¶</a></dt>
<dd><p>Get the feature of the given edges.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>edges</strong><span class="classifier">list</span></dt><dd><p>Edge indices</p>
</dd>
</dl>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><dl class="simple">
<dt>dict</dt><dd><p>The dictionary containing all relevant features.</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.get_edge_feature_names">
<code class="sig-name descname">get_edge_feature_names</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.data.GraphData.get_edge_feature_names" title="Permalink to this definition">¶</a></dt>
<dd><p>Get all the names of edge features</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.get_edge_num">
<code class="sig-name descname">get_edge_num</code><span class="sig-paren">(</span><span class="sig-paren">)</span> &#x2192; int<a class="headerlink" href="#graph4nlp.data.data.GraphData.get_edge_num" title="Permalink to this definition">¶</a></dt>
<dd><p>Get the number of edges in the graph</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>num_edges</strong><span class="classifier">int</span></dt><dd><p>The number of edges</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.get_node_attrs">
<code class="sig-name descname">get_node_attrs</code><span class="sig-paren">(</span><em class="sig-param">nodes: Union[int, slice]</em><span class="sig-paren">)</span> &#x2192; List[Any]<a class="headerlink" href="#graph4nlp.data.data.GraphData.get_node_attrs" title="Permalink to this definition">¶</a></dt>
<dd><p>Get the attributes of the given <cite>nodes</cite>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>nodes</strong><span class="classifier">int or slice</span></dt><dd><p>The given node index</p>
</dd>
</dl>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><dl class="simple">
<dt>list</dt><dd><p>The node attribute dictionary.</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.get_node_features">
<code class="sig-name descname">get_node_features</code><span class="sig-paren">(</span><em class="sig-param">nodes: Union[int, slice]</em><span class="sig-paren">)</span> &#x2192; Dict[str, torch.Tensor]<a class="headerlink" href="#graph4nlp.data.data.GraphData.get_node_features" title="Permalink to this definition">¶</a></dt>
<dd><p>Get the node feature dictionary of the <cite>nodes</cite></p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>nodes: int or slice</strong></dt><dd><p>The nodes to be accessed</p>
</dd>
</dl>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><dl class="simple">
<dt>node_features: dict</dt><dd><p>The reference dict of the actual tensor</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.get_node_num">
<code class="sig-name descname">get_node_num</code><span class="sig-paren">(</span><span class="sig-paren">)</span> &#x2192; int<a class="headerlink" href="#graph4nlp.data.data.GraphData.get_node_num" title="Permalink to this definition">¶</a></dt>
<dd><p>Get the number of nodes in the graph.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><dl class="simple">
<dt>num_nodes: int</dt><dd><p>The number of nodes in the graph.</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.node_attributes">
<em class="property">property </em><code class="sig-name descname">node_attributes</code><a class="headerlink" href="#graph4nlp.data.data.GraphData.node_attributes" title="Permalink to this definition">¶</a></dt>
<dd><p>Access node attribute dictionary</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>node_attributes</strong><span class="classifier">list</span></dt><dd><p>The list of node attributes</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.node_feature_names">
<code class="sig-name descname">node_feature_names</code><span class="sig-paren">(</span><span class="sig-paren">)</span> &#x2192; List[str]<a class="headerlink" href="#graph4nlp.data.data.GraphData.node_feature_names" title="Permalink to this definition">¶</a></dt>
<dd><p>Get the names of node features.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><dl class="simple">
<dt>List[str]</dt><dd><p>The collection of feature names.</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.node_features">
<em class="property">property </em><code class="sig-name descname">node_features</code><a class="headerlink" href="#graph4nlp.data.data.GraphData.node_features" title="Permalink to this definition">¶</a></dt>
<dd><p>Access and modify node feature vectors (tensor).
This property can be accessed in a dict-of-dict fashion,
with the order being [name][index].
‘name’ indicates the name of the feature vector. ‘index’ selects
the specific nodes to be accessed.
When accessed independently, returns the feature dictionary with
the format {name: tensor}.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><dl class="simple">
<dt>NodeFeatView</dt><dd></dd>
</dl>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">g</span> <span class="o">=</span> <span class="n">GraphData</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">g</span><span class="o">.</span><span class="n">add_nodes</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">g</span><span class="o">.</span><span class="n">node_features</span><span class="p">[</span><span class="s1">&#39;x&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">((</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">g</span><span class="o">.</span><span class="n">node_features</span><span class="p">[</span><span class="s1">&#39;x&#39;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
<span class="go">torch.Tensor([0.1036, 0.6757, 0.4702, 0.8938, 0.6337, 0.3290,</span>
<span class="go">              0.6739, 0.1091, 0.7996, 0.0586])</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.nodes">
<em class="property">property </em><code class="sig-name descname">nodes</code><a class="headerlink" href="#graph4nlp.data.data.GraphData.nodes" title="Permalink to this definition">¶</a></dt>
<dd><p>Return a node view through which the user can access the features
and attributes.</p>
<p>A NodeView object provides a high-level view of the underlying
storage of the features and supports both query and modification
to the original storage.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><dl class="simple">
<dt>node: NodeView</dt><dd><p>The node view</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.remove_all_edges">
<code class="sig-name descname">remove_all_edges</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.data.GraphData.remove_all_edges" title="Permalink to this definition">¶</a></dt>
<dd><p>Remove all the edges and the corresponding features and attributes in GraphData.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><dl class="simple">
<dt>None</dt><dd></dd>
</dl>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">g</span> <span class="o">=</span> <span class="n">GraphData</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">g</span><span class="o">.</span><span class="n">add_nodes</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">g</span><span class="o">.</span><span class="n">add_edges</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">9</span><span class="p">,</span> <span class="mi">1</span><span class="p">)),</span> <span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">1</span><span class="p">)))</span>
<span class="go"># Added some feature tensors to the edges</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">g</span><span class="o">.</span><span class="n">edge_features</span><span class="p">[</span><span class="s1">&#39;random&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">((</span><span class="mi">9</span><span class="p">,</span> <span class="mi">1024</span><span class="p">,</span> <span class="mi">1024</span><span class="p">))</span>
<span class="go"># Remove all edges and the corresponding data. The tensor memory is freed now.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">g</span><span class="o">.</span><span class="n">remove_all_edges</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.set_edge_feature">
<code class="sig-name descname">set_edge_feature</code><span class="sig-paren">(</span><em class="sig-param">edges: Union[int, slice, List[int]], new_data: Dict[str, torch.Tensor]</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.data.GraphData.set_edge_feature" title="Permalink to this definition">¶</a></dt>
<dd><p>Set edge feature</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>edges</strong><span class="classifier">int or list or slice</span></dt><dd><p>Edge indices</p>
</dd>
<dt><strong>new_data</strong><span class="classifier">dict</span></dt><dd><p>New data</p>
</dd>
</dl>
</dd>
<dt class="field-even">Raises</dt>
<dd class="field-even"><dl class="simple">
<dt>SizeMismatchException</dt><dd><p>If the size of the new features does not match the node number</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.set_node_features">
<code class="sig-name descname">set_node_features</code><span class="sig-paren">(</span><em class="sig-param">nodes: Union[int, slice], new_data: Dict[str, torch.Tensor]</em><span class="sig-paren">)</span> &#x2192; None<a class="headerlink" href="#graph4nlp.data.data.GraphData.set_node_features" title="Permalink to this definition">¶</a></dt>
<dd><p>Set the features of the <cite>nodes</cite> with the given <cite>new_data`</cite>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>nodes</strong><span class="classifier">int or slice</span></dt><dd><p>The nodes involved</p>
</dd>
<dt><strong>new_data</strong><span class="classifier">dict</span></dt><dd><p>The new data to write. Key indicates feature name and value indicates the
actual value.</p>
</dd>
</dl>
</dd>
<dt class="field-even">Raises</dt>
<dd class="field-even"><dl class="simple">
<dt>SizeMismatchException</dt><dd><p>If the size of the new features does not match the node number</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.sparse_adj">
<code class="sig-name descname">sparse_adj</code><span class="sig-paren">(</span><em class="sig-param">batch_view: bool = False</em><span class="sig-paren">)</span> &#x2192; Union[torch.Tensor, List[torch.Tensor]]<a class="headerlink" href="#graph4nlp.data.data.GraphData.sparse_adj" title="Permalink to this definition">¶</a></dt>
<dd><p>Return the scipy.sparse.coo_matrix form of the adjacency matrix
:param batch_view: Whether to return the split view of the adjacency matrix.</p>
<blockquote>
<div><p>Return a list of COO matrix if True</p>
</div></blockquote>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><dl class="simple">
<dt>torch.Tensor or list of torch.Tensor</dt><dd></dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.split_features">
<code class="sig-name descname">split_features</code><span class="sig-paren">(</span><em class="sig-param">input_tensor: torch.Tensor</em>, <em class="sig-param">type: str = 'node'</em><span class="sig-paren">)</span> &#x2192; torch.Tensor<a class="headerlink" href="#graph4nlp.data.data.GraphData.split_features" title="Permalink to this definition">¶</a></dt>
<dd><p>Convert a tensor from [N, <a href="#id5"><span class="problematic" id="id6">*</span></a>] to [B, N_max, <a href="#id7"><span class="problematic" id="id8">*</span></a>] with zero padding according
to the batch information stored in the graph.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>input_tensor</strong><span class="classifier">torch.Tensor</span></dt><dd></dd>
<dt><strong>The original tensor to be split.</strong></dt><dd></dd>
<dt><strong>type</strong><span class="classifier">str</span></dt><dd></dd>
<dt><strong>‘node’ or ‘edge’. Indicates the source of batch information.</strong></dt><dd></dd>
</dl>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><dl class="simple">
<dt>torch.Tensor</dt><dd></dd>
<dt>The split tensor.</dt><dd></dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.to">
<code class="sig-name descname">to</code><span class="sig-paren">(</span><em class="sig-param">device: str</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.data.GraphData.to" title="Permalink to this definition">¶</a></dt>
<dd><p>Move the GraphData object to different devices(cpu, gpu, etc.).
The usage of this method is similar to that of torch.Tensor and
dgl.DGLGraph</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>device: str</strong></dt><dd><p>The target device.</p>
</dd>
</dl>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><dl class="simple">
<dt>self</dt><dd></dd>
</dl>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.data.GraphData.to_dgl">
<code class="sig-name descname">to_dgl</code><span class="sig-paren">(</span><span class="sig-paren">)</span> &#x2192; dgl.heterograph.DGLHeteroGraph<a class="headerlink" href="#graph4nlp.data.data.GraphData.to_dgl" title="Permalink to this definition">¶</a></dt>
<dd><p>Convert to dgl.DGLGraph
Note that there will be some information loss when calling this function,
e.g. the batch-related information
will not be copied to DGLGraph since it is only intended for computation.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>g</strong><span class="classifier">dgl.DGLGraph</span></dt><dd><p>The converted dgl.DGLGraph</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

</dd></dl>

<dl class="function">
<dt id="graph4nlp.data.data.to_batch">
<code class="sig-prename descclassname">graph4nlp.data.data.</code><code class="sig-name descname">to_batch</code><span class="sig-paren">(</span><em class="sig-param">graphs: List[graph4nlp.pytorch.data.data.GraphData] = None</em><span class="sig-paren">)</span> &#x2192; graph4nlp.pytorch.data.data.GraphData<a class="headerlink" href="#graph4nlp.data.data.to_batch" title="Permalink to this definition">¶</a></dt>
<dd><p>Convert a list of GraphData to a large graph (a batch).</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>graphs</strong><span class="classifier">list of GraphData</span></dt><dd><p>The list of GraphData to be batched</p>
</dd>
</dl>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><dl class="simple">
<dt>GraphData</dt><dd><p>The large graph containing all the graphs in the batch.</p>
</dd>
</dl>
</dd>
</dl>
</dd></dl>

<span class="target" id="module-graph4nlp.data.dataset"></span><dl class="class">
<dt id="graph4nlp.data.dataset.Dataset">
<em class="property">class </em><code class="sig-prename descclassname">graph4nlp.data.dataset.</code><code class="sig-name descname">Dataset</code><span class="sig-paren">(</span><em class="sig-param">root</em>, <em class="sig-param">topology_builder</em>, <em class="sig-param">topology_subdir</em>, <em class="sig-param">tokenizer=&lt;function word_tokenize&gt;</em>, <em class="sig-param">lower_case=True</em>, <em class="sig-param">pretrained_word_emb_name='840B'</em>, <em class="sig-param">pretrained_word_emb_url=None</em>, <em class="sig-param">target_pretrained_word_emb_name=None</em>, <em class="sig-param">target_pretrained_word_emb_url=None</em>, <em class="sig-param">pretrained_word_emb_cache_dir='.vector_cache/'</em>, <em class="sig-param">max_word_vocab_size=None</em>, <em class="sig-param">min_word_vocab_freq=1</em>, <em class="sig-param">use_val_for_vocab=False</em>, <em class="sig-param">seed=1234</em>, <em class="sig-param">thread_number=4</em>, <em class="sig-param">port=9000</em>, <em class="sig-param">timeout=15000</em>, <em class="sig-param">for_inference=False</em>, <em class="sig-param">reused_vocab_model=None</em>, <em class="sig-param">**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Dataset" title="Permalink to this definition">¶</a></dt>
<dd><p>Base class for datasets.</p>
<p>The dataset is organized in a two-layer index style. Direct access
to the dataset object, e.g. Dataset[1], will first be converted to
the access to the internal index list, which is then passed to
access the actual data. This design is for the ease of sampling.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>root: str</strong></dt><dd><p>The root directory path where the dataset is stored.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Examples</p>
<p>Suppose we have a Dataset containing 5 data items
[‘a’, ‘b’, ‘c’, ‘d’, ‘e’]. The indices of the 5 elements in the
list are correspondingly [0, 1, 2, 3, 4]. Suppose the dataset
is shuffled, which shuffles the internal index list, the consequent
indices becomes [2, 3, 1, 4, 5]. Then an access to the dataset
<cite>Dataset[2]</cite> will first access the indices[2] which is 1, and then
use the received index to access the actual dataset, which will
return the actual data item ‘b’. Now to the user the 3rd ([2])
element in the dataset got shuffled and is not ‘c’.</p>
<dl class="field-list simple">
<dt class="field-odd">Attributes</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>processed_dir</strong></dt><dd></dd>
<dt><strong>processed_file_names</strong></dt><dd></dd>
<dt><strong>processed_file_paths</strong></dt><dd></dd>
<dt><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.raw_dir" title="graph4nlp.data.dataset.Dataset.raw_dir"><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_dir</span></code></a></dt><dd><p>The directory where the raw data is stored.</p>
</dd>
<dt><strong>raw_file_names</strong></dt><dd></dd>
<dt><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.raw_file_paths" title="graph4nlp.data.dataset.Dataset.raw_file_paths"><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_file_paths</span></code></a></dt><dd><p>The paths to raw files.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Methods</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.build_topology" title="graph4nlp.data.dataset.Dataset.build_topology"><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_topology</span></code></a>(data_items)</p></td>
<td><p>Build graph topology for each item in the dataset.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.build_vocab" title="graph4nlp.data.dataset.Dataset.build_vocab"><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_vocab</span></code></a>()</p></td>
<td><p>Build the vocabulary.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.collate_fn" title="graph4nlp.data.dataset.Dataset.collate_fn"><code class="xref py py-obj docutils literal notranslate"><span class="pre">collate_fn</span></code></a>(data_list)</p></td>
<td><p>Takes a list of data and convert it to a batch of data.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.download" title="graph4nlp.data.dataset.Dataset.download"><code class="xref py py-obj docutils literal notranslate"><span class="pre">download</span></code></a>()</p></td>
<td><p>Download the raw data from the Internet.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.parse_file" title="graph4nlp.data.dataset.Dataset.parse_file"><code class="xref py py-obj docutils literal notranslate"><span class="pre">parse_file</span></code></a>(file_path)</p></td>
<td><p>To be implemented in task-specific dataset base class.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.read_raw_data" title="graph4nlp.data.dataset.Dataset.read_raw_data"><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_raw_data</span></code></a>()</p></td>
<td><p>Read raw data from the disk and put them in a dictionary (<cite>self.data</cite>).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.vectorization" title="graph4nlp.data.dataset.Dataset.vectorization"><code class="xref py py-obj docutils literal notranslate"><span class="pre">vectorization</span></code></a>(data_items)</p></td>
<td><p>Convert tokens to indices which can be processed by downstream models.</p></td>
</tr>
</tbody>
</table>
<table class="docutils align-default">
<colgroup>
<col style="width: 77%" />
<col style="width: 23%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><strong>process_data_items</strong></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p><strong>register_datapipe_as_function</strong></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p><strong>register_function</strong></p></td>
<td></td>
</tr>
</tbody>
</table>
<dl class="method">
<dt id="graph4nlp.data.dataset.Dataset.build_topology">
<code class="sig-name descname">build_topology</code><span class="sig-paren">(</span><em class="sig-param">data_items</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Dataset.build_topology" title="Permalink to this definition">¶</a></dt>
<dd><p>Build graph topology for each item in the dataset. The generated graph
is bound to the <cite>graph</cite> attribute of the DataItem.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Dataset.build_vocab">
<code class="sig-name descname">build_vocab</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Dataset.build_vocab" title="Permalink to this definition">¶</a></dt>
<dd><p>Build the vocabulary. If <cite>self.use_val_for_vocab</cite> is <cite>True</cite>,
use both training set and validation set for building
the vocabulary. Otherwise only the training set is used.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Dataset.collate_fn">
<em class="property">abstract static </em><code class="sig-name descname">collate_fn</code><span class="sig-paren">(</span><em class="sig-param">data_list</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Dataset.collate_fn" title="Permalink to this definition">¶</a></dt>
<dd><p>Takes a list of data and convert it to a batch of data.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Dataset.download">
<em class="property">abstract </em><code class="sig-name descname">download</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Dataset.download" title="Permalink to this definition">¶</a></dt>
<dd><p>Download the raw data from the Internet.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Dataset.parse_file">
<em class="property">abstract </em><code class="sig-name descname">parse_file</code><span class="sig-paren">(</span><em class="sig-param">file_path</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Dataset.parse_file" title="Permalink to this definition">¶</a></dt>
<dd><p>To be implemented in task-specific dataset base class.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Dataset.raw_dir">
<em class="property">property </em><code class="sig-name descname">raw_dir</code><a class="headerlink" href="#graph4nlp.data.dataset.Dataset.raw_dir" title="Permalink to this definition">¶</a></dt>
<dd><p>The directory where the raw data is stored.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Dataset.raw_file_paths">
<em class="property">property </em><code class="sig-name descname">raw_file_paths</code><a class="headerlink" href="#graph4nlp.data.dataset.Dataset.raw_file_paths" title="Permalink to this definition">¶</a></dt>
<dd><p>The paths to raw files.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Dataset.read_raw_data">
<code class="sig-name descname">read_raw_data</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Dataset.read_raw_data" title="Permalink to this definition">¶</a></dt>
<dd><p>Read raw data from the disk and put them in a dictionary (<cite>self.data</cite>).
The raw data file should be organized as the format defined in <cite>self.parse_file()</cite> method.</p>
<p>This function calls <cite>self.parse_file()</cite> repeatedly and pass the file paths in
<cite>self.raw_file_names</cite> once at a time.</p>
<p>This function builds <cite>self.data</cite> which is a dict of {int (index): DataItem},
where the id represents the index of the DataItem w.r.t. the whole dataset.</p>
<p>This function also builds the <cite>self.split_ids</cite> dictionary whose keys correspond to
those of self.raw_file_names defined by the user, indicating the indices of each
subset (e.g. train, val and test).</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Dataset.vectorization">
<em class="property">abstract </em><code class="sig-name descname">vectorization</code><span class="sig-paren">(</span><em class="sig-param">data_items</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Dataset.vectorization" title="Permalink to this definition">¶</a></dt>
<dd><p>Convert tokens to indices which can be processed by downstream models.</p>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="graph4nlp.data.dataset.Text2TextDataset">
<em class="property">class </em><code class="sig-prename descclassname">graph4nlp.data.dataset.</code><code class="sig-name descname">Text2TextDataset</code><span class="sig-paren">(</span><em class="sig-param">graph_name: str</em>, <em class="sig-param">root_dir: str = None</em>, <em class="sig-param">static_or_dynamic: str = None</em>, <em class="sig-param">topology_builder: Union[graph4nlp.pytorch.modules.graph_construction.base.StaticGraphConstructionBase</em>, <em class="sig-param">graph4nlp.pytorch.modules.graph_construction.base.DynamicGraphConstructionBase] = &lt;class 'graph4nlp.pytorch.modules.graph_construction.dependency_graph_construction.DependencyBasedGraphConstruction'&gt;</em>, <em class="sig-param">topology_subdir: str = None</em>, <em class="sig-param">dynamic_init_graph_name: str = None</em>, <em class="sig-param">dynamic_init_topology_builder: graph4nlp.pytorch.modules.graph_construction.base.StaticGraphConstructionBase = None</em>, <em class="sig-param">dynamic_init_topology_aux_args=None</em>, <em class="sig-param">share_vocab=True</em>, <em class="sig-param">**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Text2TextDataset" title="Permalink to this definition">¶</a></dt>
<dd><blockquote>
<div><p>The dataset for text-to-text applications.</p>
</div></blockquote>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>graph_name</strong> (<em>str</em>) – <p>The name of graph construction method. E.g., “dependency”.
Note that if it is in the provided graph names (i.e., “dependency”,             “constituency”, “ie”, “node_emb”, “node_emb_refine”), the following             parameters are set by default and users can’t modify them:</p>
<blockquote>
<div><ol class="arabic simple">
<li><p><code class="docutils literal notranslate"><span class="pre">topology_builder</span></code></p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">static_or_dynamic</span></code></p></li>
</ol>
</div></blockquote>
<p>If you need to customize your graph construction method, you should rename the             <code class="docutils literal notranslate"><span class="pre">graph_name</span></code> and set the parameters above.</p>
</p></li>
<li><p><strong>root_dir</strong> (<em>str</em><em>, </em><em>default=None</em>) – The path of dataset.</p></li>
<li><p><strong>topology_builder</strong> (<em>Union</em><em>[</em><em>StaticGraphConstructionBase</em><em>, </em><em>DynamicGraphConstructionBase</em><em>]</em><em>, </em><em>default=None</em>) – The graph construction class.</p></li>
<li><p><strong>topology_subdir</strong> (<em>str</em>) – The directory name of processed path.</p></li>
<li><p><strong>static_or_dynamic</strong> (<em>str</em><em>, </em><em>default='static'</em>) – The graph type. Expected in (‘static’, ‘dynamic’)</p></li>
<li><p><strong>dynamic_init_graph_name</strong> (<em>str</em><em>, </em><em>default=None</em>) – <p>The graph name of the initial graph. Expected in (None, “line”,             “dependency”, “constituency”).
Note that if it is in the provided graph names (i.e., “line”, “dependency”,             “constituency”), the following parameters are set by default and users             can’t modify them:</p>
<blockquote>
<div><ol class="arabic simple">
<li><p><code class="docutils literal notranslate"><span class="pre">dynamic_init_topology_builder</span></code></p></li>
</ol>
</div></blockquote>
<p>If you need to customize your graph construction method, you should rename the             <code class="docutils literal notranslate"><span class="pre">graph_name</span></code> and set the parameters above.</p>
</p></li>
<li><p><strong>dynamic_init_topology_builder</strong> (<em>StaticGraphConstructionBase</em>) – The graph construction class.</p></li>
<li><p><strong>dynamic_init_topology_aux_args</strong> (<em>None</em><em>,</em>) – TBD.</p></li>
</ul>
</dd>
<dt class="field-even">Attributes</dt>
<dd class="field-even"><dl class="simple">
<dt><strong>processed_dir</strong></dt><dd></dd>
<dt><strong>processed_file_names</strong></dt><dd></dd>
<dt><strong>processed_file_paths</strong></dt><dd></dd>
<dt><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_dir</span></code></dt><dd><p>The directory where the raw data is stored.</p>
</dd>
<dt><strong>raw_file_names</strong></dt><dd></dd>
<dt><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_file_paths</span></code></dt><dd><p>The paths to raw files.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Methods</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_topology</span></code>(data_items)</p></td>
<td><p>Build graph topology for each item in the dataset.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_vocab</span></code>()</p></td>
<td><p>Build the vocabulary.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Text2TextDataset.collate_fn" title="graph4nlp.data.dataset.Text2TextDataset.collate_fn"><code class="xref py py-obj docutils literal notranslate"><span class="pre">collate_fn</span></code></a>(data_list)</p></td>
<td><p>Takes a list of data and convert it to a batch of data.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">download</span></code>()</p></td>
<td><p>Download the raw data from the Internet.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Text2TextDataset.parse_file" title="graph4nlp.data.dataset.Text2TextDataset.parse_file"><code class="xref py py-obj docutils literal notranslate"><span class="pre">parse_file</span></code></a>(file_path)</p></td>
<td><p>Read and parse the file specified by <cite>file_path</cite>.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_raw_data</span></code>()</p></td>
<td><p>Read raw data from the disk and put them in a dictionary (<cite>self.data</cite>).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Text2TextDataset.vectorization" title="graph4nlp.data.dataset.Text2TextDataset.vectorization"><code class="xref py py-obj docutils literal notranslate"><span class="pre">vectorization</span></code></a>(data_items)</p></td>
<td><p>Convert tokens to indices which can be processed by downstream models.</p></td>
</tr>
</tbody>
</table>
<table class="docutils align-default">
<colgroup>
<col style="width: 77%" />
<col style="width: 23%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><strong>process_data_items</strong></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p><strong>register_datapipe_as_function</strong></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p><strong>register_function</strong></p></td>
<td></td>
</tr>
</tbody>
</table>
<dl class="method">
<dt id="graph4nlp.data.dataset.Text2TextDataset.collate_fn">
<em class="property">static </em><code class="sig-name descname">collate_fn</code><span class="sig-paren">(</span><em class="sig-param">data_list: [&lt;class 'graph4nlp.pytorch.data.dataset.Text2TextDataItem'&gt;]</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Text2TextDataset.collate_fn" title="Permalink to this definition">¶</a></dt>
<dd><p>Takes a list of data and convert it to a batch of data.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Text2TextDataset.parse_file">
<code class="sig-name descname">parse_file</code><span class="sig-paren">(</span><em class="sig-param">file_path</em><span class="sig-paren">)</span> &#x2192; list<a class="headerlink" href="#graph4nlp.data.dataset.Text2TextDataset.parse_file" title="Permalink to this definition">¶</a></dt>
<dd><p>Read and parse the file specified by <cite>file_path</cite>. The file format is
specified by each individual task-specific base class. Returns all
the indices of data items in this file w.r.t. the whole dataset.</p>
<p>For Text2TextDataset, the format of the input file should contain
lines of input, each line representing one record of data. The
input and output is separated by a tab( ).</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>file_path: str</strong></dt><dd><p>The path of the input file.</p>
</dd>
</dl>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><dl class="simple">
<dt>list</dt><dd><p>The indices of data items in the file w.r.t. the whole dataset.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Examples</p>
<p>input: list job use languageid0 job ( ANS ) , language ( ANS , languageid0 )</p>
<dl class="simple">
<dt>DataItem:</dt><dd><p>input_text=”list job use languageid0”,
output_text=”job ( ANS ) , language ( ANS , languageid0 )”</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Text2TextDataset.vectorization">
<code class="sig-name descname">vectorization</code><span class="sig-paren">(</span><em class="sig-param">data_items</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Text2TextDataset.vectorization" title="Permalink to this definition">¶</a></dt>
<dd><p>Convert tokens to indices which can be processed by downstream models.</p>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="graph4nlp.data.dataset.Text2TreeDataset">
<em class="property">class </em><code class="sig-prename descclassname">graph4nlp.data.dataset.</code><code class="sig-name descname">Text2TreeDataset</code><span class="sig-paren">(</span><em class="sig-param">graph_name: str</em>, <em class="sig-param">root_dir: str = None</em>, <em class="sig-param">static_or_dynamic: str = None</em>, <em class="sig-param">topology_builder: Union[graph4nlp.pytorch.modules.graph_construction.base.StaticGraphConstructionBase</em>, <em class="sig-param">graph4nlp.pytorch.modules.graph_construction.base.DynamicGraphConstructionBase] = &lt;class 'graph4nlp.pytorch.modules.graph_construction.dependency_graph_construction.DependencyBasedGraphConstruction'&gt;</em>, <em class="sig-param">topology_subdir: str = None</em>, <em class="sig-param">dynamic_init_graph_name: str = None</em>, <em class="sig-param">dynamic_init_topology_builder: graph4nlp.pytorch.modules.graph_construction.base.StaticGraphConstructionBase = None</em>, <em class="sig-param">dynamic_init_topology_aux_args=None</em>, <em class="sig-param">share_vocab=True</em>, <em class="sig-param">**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Text2TreeDataset" title="Permalink to this definition">¶</a></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Attributes</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>processed_dir</strong></dt><dd></dd>
<dt><strong>processed_file_names</strong></dt><dd></dd>
<dt><strong>processed_file_paths</strong></dt><dd></dd>
<dt><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_dir</span></code></dt><dd><p>The directory where the raw data is stored.</p>
</dd>
<dt><strong>raw_file_names</strong></dt><dd></dd>
<dt><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_file_paths</span></code></dt><dd><p>The paths to raw files.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Methods</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_topology</span></code>(data_items)</p></td>
<td><p>Build graph topology for each item in the dataset.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Text2TreeDataset.build_vocab" title="graph4nlp.data.dataset.Text2TreeDataset.build_vocab"><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_vocab</span></code></a>()</p></td>
<td><p>Build the vocabulary.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Text2TreeDataset.collate_fn" title="graph4nlp.data.dataset.Text2TreeDataset.collate_fn"><code class="xref py py-obj docutils literal notranslate"><span class="pre">collate_fn</span></code></a>(data_list)</p></td>
<td><p>Takes a list of data and convert it to a batch of data.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">download</span></code>()</p></td>
<td><p>Download the raw data from the Internet.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Text2TreeDataset.parse_file" title="graph4nlp.data.dataset.Text2TreeDataset.parse_file"><code class="xref py py-obj docutils literal notranslate"><span class="pre">parse_file</span></code></a>(file_path)</p></td>
<td><p>Read and parse the file specified by <cite>file_path</cite>.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_raw_data</span></code>()</p></td>
<td><p>Read raw data from the disk and put them in a dictionary (<cite>self.data</cite>).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Text2TreeDataset.vectorization" title="graph4nlp.data.dataset.Text2TreeDataset.vectorization"><code class="xref py py-obj docutils literal notranslate"><span class="pre">vectorization</span></code></a>(data_items)</p></td>
<td><p>For tree decoder we also need the vectorize the tree output.</p></td>
</tr>
</tbody>
</table>
<table class="docutils align-default">
<colgroup>
<col style="width: 77%" />
<col style="width: 23%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><strong>process_data_items</strong></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p><strong>register_datapipe_as_function</strong></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p><strong>register_function</strong></p></td>
<td></td>
</tr>
</tbody>
</table>
<dl class="method">
<dt id="graph4nlp.data.dataset.Text2TreeDataset.build_vocab">
<code class="sig-name descname">build_vocab</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Text2TreeDataset.build_vocab" title="Permalink to this definition">¶</a></dt>
<dd><p>Build the vocabulary. If <cite>self.use_val_for_vocab</cite> is <cite>True</cite>,
use both training set and validation set for building
the vocabulary. Otherwise only the training set is used.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Text2TreeDataset.collate_fn">
<em class="property">static </em><code class="sig-name descname">collate_fn</code><span class="sig-paren">(</span><em class="sig-param">data_list: [&lt;class 'graph4nlp.pytorch.data.dataset.Text2TreeDataItem'&gt;]</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Text2TreeDataset.collate_fn" title="Permalink to this definition">¶</a></dt>
<dd><p>Takes a list of data and convert it to a batch of data.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Text2TreeDataset.parse_file">
<code class="sig-name descname">parse_file</code><span class="sig-paren">(</span><em class="sig-param">file_path</em><span class="sig-paren">)</span> &#x2192; list<a class="headerlink" href="#graph4nlp.data.dataset.Text2TreeDataset.parse_file" title="Permalink to this definition">¶</a></dt>
<dd><p>Read and parse the file specified by <cite>file_path</cite>. The file format is specified by
each individual task-specific base class. Returns all the indices of data items
in this file w.r.t. the whole dataset.</p>
<p>For Text2TreeDataset, the format of the input file should contain lines of input,
each line representing one record of data. The input and output is separated by
a tab(  ).</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>file_path: str</strong></dt><dd><p>The path of the input file.</p>
</dd>
</dl>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><dl class="simple">
<dt>list</dt><dd><p>The indices of data items in the file w.r.t. the whole dataset.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Examples</p>
<p>input: list job use languageid0 job ( ANS ) , language ( ANS , languageid0 )</p>
<p>DataItem: input_text=”list job use languageid0”, output_text=”job ( ANS ) ,
language ( ANS , languageid0 )”</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Text2TreeDataset.vectorization">
<code class="sig-name descname">vectorization</code><span class="sig-paren">(</span><em class="sig-param">data_items</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Text2TreeDataset.vectorization" title="Permalink to this definition">¶</a></dt>
<dd><p>For tree decoder we also need the vectorize the tree output.</p>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="graph4nlp.data.dataset.Text2LabelDataset">
<em class="property">class </em><code class="sig-prename descclassname">graph4nlp.data.dataset.</code><code class="sig-name descname">Text2LabelDataset</code><span class="sig-paren">(</span><em class="sig-param">graph_name: str</em>, <em class="sig-param">root_dir: str = None</em>, <em class="sig-param">static_or_dynamic: str = None</em>, <em class="sig-param">topology_builder: Union[graph4nlp.pytorch.modules.graph_construction.base.StaticGraphConstructionBase</em>, <em class="sig-param">graph4nlp.pytorch.modules.graph_construction.base.DynamicGraphConstructionBase] = &lt;class 'graph4nlp.pytorch.modules.graph_construction.dependency_graph_construction.DependencyBasedGraphConstruction'&gt;</em>, <em class="sig-param">topology_subdir: str = None</em>, <em class="sig-param">dynamic_init_graph_name: str = None</em>, <em class="sig-param">dynamic_init_topology_builder: graph4nlp.pytorch.modules.graph_construction.base.StaticGraphConstructionBase = None</em>, <em class="sig-param">dynamic_init_topology_aux_args=None</em>, <em class="sig-param">**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Text2LabelDataset" title="Permalink to this definition">¶</a></dt>
<dd><p>The dataset for text-to-label applications.
:param graph_name: The name of graph construction method. E.g., “dependency”.</p>
<blockquote>
<div><dl class="simple">
<dt>Note that if it is in the provided graph names (i.e., “dependency”,             “constituency”, “ie”, “node_emb”, “node_emb_refine”), the following             parameters are set by default and users can’t modify them:</dt><dd><ol class="arabic simple">
<li><p><code class="docutils literal notranslate"><span class="pre">topology_builder</span></code></p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">static_or_dynamic</span></code></p></li>
</ol>
</dd>
</dl>
<p>If you need to customize your graph construction method, you should rename the             <code class="docutils literal notranslate"><span class="pre">graph_name</span></code> and set the parameters above.</p>
</div></blockquote>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>root_dir</strong> (<em>str</em><em>, </em><em>default=None</em>) – The path of dataset.</p></li>
<li><p><strong>topology_builder</strong> (<em>Union</em><em>[</em><em>StaticGraphConstructionBase</em><em>, </em><em>DynamicGraphConstructionBase</em><em>]</em><em>, </em><em>default=None</em>) – The graph construction class.</p></li>
<li><p><strong>topology_subdir</strong> (<em>str</em>) – The directory name of processed path.</p></li>
<li><p><strong>static_or_dynamic</strong> (<em>str</em><em>, </em><em>default='static'</em>) – The graph type. Expected in (‘static’, ‘dynamic’)</p></li>
<li><p><strong>dynamic_init_graph_name</strong> (<em>str</em><em>, </em><em>default=None</em>) – <p>The graph name of the initial graph. Expected in (None, “line”,             “dependency”, “constituency”).
Note that if it is in the provided graph names (i.e., “line”, “dependency”,             “constituency”), the following parameters are set by default and users             can’t modify them:</p>
<blockquote>
<div><ol class="arabic simple">
<li><p><code class="docutils literal notranslate"><span class="pre">dynamic_init_topology_builder</span></code></p></li>
</ol>
</div></blockquote>
<p>If you need to customize your graph construction method, you should rename the             <code class="docutils literal notranslate"><span class="pre">graph_name</span></code> and set the parameters above.</p>
</p></li>
<li><p><strong>dynamic_init_topology_builder</strong> (<em>StaticGraphConstructionBase</em>) – The graph construction class.</p></li>
<li><p><strong>dynamic_init_topology_aux_args</strong> (<em>None</em><em>,</em>) – TBD.</p></li>
</ul>
</dd>
<dt class="field-even">Attributes</dt>
<dd class="field-even"><dl class="simple">
<dt><strong>processed_dir</strong></dt><dd></dd>
<dt><strong>processed_file_names</strong></dt><dd></dd>
<dt><strong>processed_file_paths</strong></dt><dd></dd>
<dt><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_dir</span></code></dt><dd><p>The directory where the raw data is stored.</p>
</dd>
<dt><strong>raw_file_names</strong></dt><dd></dd>
<dt><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_file_paths</span></code></dt><dd><p>The paths to raw files.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Methods</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_topology</span></code>(data_items)</p></td>
<td><p>Build graph topology for each item in the dataset.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Text2LabelDataset.build_vocab" title="graph4nlp.data.dataset.Text2LabelDataset.build_vocab"><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_vocab</span></code></a>()</p></td>
<td><p>Build the vocabulary.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Text2LabelDataset.collate_fn" title="graph4nlp.data.dataset.Text2LabelDataset.collate_fn"><code class="xref py py-obj docutils literal notranslate"><span class="pre">collate_fn</span></code></a>(data_list)</p></td>
<td><p>Takes a list of data and convert it to a batch of data.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">download</span></code>()</p></td>
<td><p>Download the raw data from the Internet.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Text2LabelDataset.parse_file" title="graph4nlp.data.dataset.Text2LabelDataset.parse_file"><code class="xref py py-obj docutils literal notranslate"><span class="pre">parse_file</span></code></a>(file_path)</p></td>
<td><p>Read and parse the file specified by <cite>file_path</cite>.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_raw_data</span></code>()</p></td>
<td><p>Read raw data from the disk and put them in a dictionary (<cite>self.data</cite>).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Text2LabelDataset.vectorization" title="graph4nlp.data.dataset.Text2LabelDataset.vectorization"><code class="xref py py-obj docutils literal notranslate"><span class="pre">vectorization</span></code></a>(data_items)</p></td>
<td><p>Convert tokens to indices which can be processed by downstream models.</p></td>
</tr>
</tbody>
</table>
<table class="docutils align-default">
<colgroup>
<col style="width: 77%" />
<col style="width: 23%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><strong>process_data_items</strong></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p><strong>register_datapipe_as_function</strong></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p><strong>register_function</strong></p></td>
<td></td>
</tr>
</tbody>
</table>
<dl class="method">
<dt id="graph4nlp.data.dataset.Text2LabelDataset.build_vocab">
<code class="sig-name descname">build_vocab</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Text2LabelDataset.build_vocab" title="Permalink to this definition">¶</a></dt>
<dd><p>Build the vocabulary. If <cite>self.use_val_for_vocab</cite> is <cite>True</cite>,
use both training set and validation set for building
the vocabulary. Otherwise only the training set is used.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Text2LabelDataset.collate_fn">
<em class="property">static </em><code class="sig-name descname">collate_fn</code><span class="sig-paren">(</span><em class="sig-param">data_list: [&lt;class 'graph4nlp.pytorch.data.dataset.Text2LabelDataItem'&gt;]</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Text2LabelDataset.collate_fn" title="Permalink to this definition">¶</a></dt>
<dd><p>Takes a list of data and convert it to a batch of data.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Text2LabelDataset.parse_file">
<code class="sig-name descname">parse_file</code><span class="sig-paren">(</span><em class="sig-param">file_path</em><span class="sig-paren">)</span> &#x2192; list<a class="headerlink" href="#graph4nlp.data.dataset.Text2LabelDataset.parse_file" title="Permalink to this definition">¶</a></dt>
<dd><p>Read and parse the file specified by <cite>file_path</cite>. The file format
is specified by each individual task-specific base class. Returns
all the indices of data items in this file w.r.t. the whole dataset.</p>
<p>For Text2LabelDataset, the format of the input file should contain
lines of input, each line representing one record of data. The
input and output is separated by a tab( ).</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>file_path: str</strong></dt><dd><p>The path of the input file.</p>
</dd>
</dl>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><dl class="simple">
<dt>list</dt><dd><p>The indices of data items in the file w.r.t. the whole dataset.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Examples</p>
<p>input: How far is it from Denver to Aspen ?    NUM</p>
<p>DataItem: input_text=”How far is it from Denver to Aspen ?”, output_label=”NUM”</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.Text2LabelDataset.vectorization">
<code class="sig-name descname">vectorization</code><span class="sig-paren">(</span><em class="sig-param">data_items</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.Text2LabelDataset.vectorization" title="Permalink to this definition">¶</a></dt>
<dd><p>Convert tokens to indices which can be processed by downstream models.</p>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="graph4nlp.data.dataset.DoubleText2TextDataset">
<em class="property">class </em><code class="sig-prename descclassname">graph4nlp.data.dataset.</code><code class="sig-name descname">DoubleText2TextDataset</code><span class="sig-paren">(</span><em class="sig-param">graph_name: str</em>, <em class="sig-param">root_dir: str = None</em>, <em class="sig-param">static_or_dynamic: str = None</em>, <em class="sig-param">topology_builder: Union[graph4nlp.pytorch.modules.graph_construction.base.StaticGraphConstructionBase</em>, <em class="sig-param">graph4nlp.pytorch.modules.graph_construction.base.DynamicGraphConstructionBase] = &lt;class 'graph4nlp.pytorch.modules.graph_construction.dependency_graph_construction.DependencyBasedGraphConstruction'&gt;</em>, <em class="sig-param">topology_subdir: str = None</em>, <em class="sig-param">dynamic_init_graph_name: str = None</em>, <em class="sig-param">dynamic_init_topology_builder: graph4nlp.pytorch.modules.graph_construction.base.StaticGraphConstructionBase = None</em>, <em class="sig-param">dynamic_init_topology_aux_args=None</em>, <em class="sig-param">share_vocab=True</em>, <em class="sig-param">**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.DoubleText2TextDataset" title="Permalink to this definition">¶</a></dt>
<dd><blockquote>
<div><p>The dataset for double-text-to-text applications.</p>
</div></blockquote>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>graph_name</strong> (<em>str</em>) – <p>The name of graph construction method. E.g., “dependency”.
Note that if it is in the provided graph names (i.e., “dependency”,             “constituency”, “ie”, “node_emb”, “node_emb_refine”), the following             parameters are set by default and users can’t modify them:</p>
<blockquote>
<div><ol class="arabic simple">
<li><p><code class="docutils literal notranslate"><span class="pre">topology_builder</span></code></p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">static_or_dynamic</span></code></p></li>
</ol>
</div></blockquote>
<p>If you need to customize your graph construction method, you should rename the             <code class="docutils literal notranslate"><span class="pre">graph_name</span></code> and set the parameters above.</p>
</p></li>
<li><p><strong>root_dir</strong> (<em>str</em><em>, </em><em>default=None</em>) – The path of dataset.</p></li>
<li><p><strong>topology_builder</strong> (<em>Union</em><em>[</em><em>StaticGraphConstructionBase</em><em>, </em><em>DynamicGraphConstructionBase</em><em>]</em><em>, </em><em>default=None</em>) – The graph construction class.</p></li>
<li><p><strong>topology_subdir</strong> (<em>str</em>) – The directory name of processed path.</p></li>
<li><p><strong>static_or_dynamic</strong> (<em>str</em><em>, </em><em>default='static'</em>) – The graph type. Expected in (‘static’, ‘dynamic’)</p></li>
<li><p><strong>dynamic_init_graph_name</strong> (<em>str</em><em>, </em><em>default=None</em>) – <p>The graph name of the initial graph. Expected in (None, “line”,             “dependency”, “constituency”).
Note that if it is in the provided graph names (i.e., “line”, “dependency”,             “constituency”), the following parameters are set by default and users             can’t modify them:</p>
<blockquote>
<div><ol class="arabic simple">
<li><p><code class="docutils literal notranslate"><span class="pre">dynamic_init_topology_builder</span></code></p></li>
</ol>
</div></blockquote>
<p>If you need to customize your graph construction method, you should rename the             <code class="docutils literal notranslate"><span class="pre">graph_name</span></code> and set the parameters above.</p>
</p></li>
<li><p><strong>dynamic_init_topology_builder</strong> (<em>StaticGraphConstructionBase</em>) – The graph construction class.</p></li>
<li><p><strong>dynamic_init_topology_aux_args</strong> (<em>None</em><em>,</em>) – TBD.</p></li>
</ul>
</dd>
<dt class="field-even">Attributes</dt>
<dd class="field-even"><dl class="simple">
<dt><strong>processed_dir</strong></dt><dd></dd>
<dt><strong>processed_file_names</strong></dt><dd></dd>
<dt><strong>processed_file_paths</strong></dt><dd></dd>
<dt><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_dir</span></code></dt><dd><p>The directory where the raw data is stored.</p>
</dd>
<dt><strong>raw_file_names</strong></dt><dd></dd>
<dt><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_file_paths</span></code></dt><dd><p>The paths to raw files.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Methods</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_topology</span></code>(data_items)</p></td>
<td><p>Build graph topology for each item in the dataset.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_vocab</span></code>()</p></td>
<td><p>Build the vocabulary.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.DoubleText2TextDataset.collate_fn" title="graph4nlp.data.dataset.DoubleText2TextDataset.collate_fn"><code class="xref py py-obj docutils literal notranslate"><span class="pre">collate_fn</span></code></a>(data_list)</p></td>
<td><p>Takes a list of data and convert it to a batch of data.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">download</span></code>()</p></td>
<td><p>Download the raw data from the Internet.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.DoubleText2TextDataset.parse_file" title="graph4nlp.data.dataset.DoubleText2TextDataset.parse_file"><code class="xref py py-obj docutils literal notranslate"><span class="pre">parse_file</span></code></a>(file_path)</p></td>
<td><p>Read and parse the file specified by <cite>file_path</cite>.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_raw_data</span></code>()</p></td>
<td><p>Read raw data from the disk and put them in a dictionary (<cite>self.data</cite>).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.DoubleText2TextDataset.vectorization" title="graph4nlp.data.dataset.DoubleText2TextDataset.vectorization"><code class="xref py py-obj docutils literal notranslate"><span class="pre">vectorization</span></code></a>(data_items)</p></td>
<td><p>Convert tokens to indices which can be processed by downstream models.</p></td>
</tr>
</tbody>
</table>
<table class="docutils align-default">
<colgroup>
<col style="width: 77%" />
<col style="width: 23%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><strong>process_data_items</strong></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p><strong>register_datapipe_as_function</strong></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p><strong>register_function</strong></p></td>
<td></td>
</tr>
</tbody>
</table>
<dl class="method">
<dt id="graph4nlp.data.dataset.DoubleText2TextDataset.collate_fn">
<em class="property">static </em><code class="sig-name descname">collate_fn</code><span class="sig-paren">(</span><em class="sig-param">data_list: [&lt;class 'graph4nlp.pytorch.data.dataset.DoubleText2TextDataItem'&gt;]</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.DoubleText2TextDataset.collate_fn" title="Permalink to this definition">¶</a></dt>
<dd><p>Takes a list of data and convert it to a batch of data.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.DoubleText2TextDataset.parse_file">
<code class="sig-name descname">parse_file</code><span class="sig-paren">(</span><em class="sig-param">file_path</em><span class="sig-paren">)</span> &#x2192; list<a class="headerlink" href="#graph4nlp.data.dataset.DoubleText2TextDataset.parse_file" title="Permalink to this definition">¶</a></dt>
<dd><p>Read and parse the file specified by <cite>file_path</cite>. The file format is
specified by each individual task-specific base class.
Returns all the indices of data items in this file w.r.t. the whole dataset.</p>
<p>For DoubleText2TextDataset, the format of the input file should
contain lines of input, each line representing one record of data.
The input and output is separated by a tab(     ).
# TODO: update example</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>file_path: str</strong></dt><dd><p>The path of the input file.</p>
</dd>
</dl>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><dl class="simple">
<dt>list</dt><dd><p>The indices of data items in the file w.r.t. the whole dataset.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Examples</p>
<p>input: list job use languageid0 job ( ANS ) , language ( ANS , languageid0 )</p>
<dl class="simple">
<dt>DataItem:</dt><dd><p>input_text=”list job use languageid0”,
input_text2=”list job use languageid0”,
output_text=”job ( ANS ) , language ( ANS , languageid0 )”</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.DoubleText2TextDataset.vectorization">
<code class="sig-name descname">vectorization</code><span class="sig-paren">(</span><em class="sig-param">data_items</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.DoubleText2TextDataset.vectorization" title="Permalink to this definition">¶</a></dt>
<dd><p>Convert tokens to indices which can be processed by downstream models.</p>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="graph4nlp.data.dataset.SequenceLabelingDataset">
<em class="property">class </em><code class="sig-prename descclassname">graph4nlp.data.dataset.</code><code class="sig-name descname">SequenceLabelingDataset</code><span class="sig-paren">(</span><em class="sig-param">graph_name: str</em>, <em class="sig-param">root_dir: str = None</em>, <em class="sig-param">static_or_dynamic: str = None</em>, <em class="sig-param">topology_builder: Union[graph4nlp.pytorch.modules.graph_construction.base.StaticGraphConstructionBase</em>, <em class="sig-param">graph4nlp.pytorch.modules.graph_construction.base.DynamicGraphConstructionBase] = &lt;class 'graph4nlp.pytorch.modules.graph_construction.dependency_graph_construction.DependencyBasedGraphConstruction'&gt;</em>, <em class="sig-param">topology_subdir: str = None</em>, <em class="sig-param">tag_types: str = None</em>, <em class="sig-param">dynamic_init_graph_name: str = None</em>, <em class="sig-param">dynamic_init_topology_builder: graph4nlp.pytorch.modules.graph_construction.base.StaticGraphConstructionBase = None</em>, <em class="sig-param">**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.SequenceLabelingDataset" title="Permalink to this definition">¶</a></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Attributes</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>processed_dir</strong></dt><dd></dd>
<dt><strong>processed_file_names</strong></dt><dd></dd>
<dt><strong>processed_file_paths</strong></dt><dd></dd>
<dt><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_dir</span></code></dt><dd><p>The directory where the raw data is stored.</p>
</dd>
<dt><strong>raw_file_names</strong></dt><dd></dd>
<dt><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_file_paths</span></code></dt><dd><p>The paths to raw files.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Methods</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_topology</span></code>(data_items)</p></td>
<td><p>Build graph topology for each item in the dataset.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.SequenceLabelingDataset.build_vocab" title="graph4nlp.data.dataset.SequenceLabelingDataset.build_vocab"><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_vocab</span></code></a>()</p></td>
<td><p>Build the vocabulary.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.SequenceLabelingDataset.collate_fn" title="graph4nlp.data.dataset.SequenceLabelingDataset.collate_fn"><code class="xref py py-obj docutils literal notranslate"><span class="pre">collate_fn</span></code></a>(data_list)</p></td>
<td><p>Takes a list of data and convert it to a batch of data.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">download</span></code>()</p></td>
<td><p>Download the raw data from the Internet.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.SequenceLabelingDataset.parse_file" title="graph4nlp.data.dataset.SequenceLabelingDataset.parse_file"><code class="xref py py-obj docutils literal notranslate"><span class="pre">parse_file</span></code></a>(file_path)</p></td>
<td><p>Read and parse the file specified by <cite>file_path</cite>.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_raw_data</span></code>()</p></td>
<td><p>Read raw data from the disk and put them in a dictionary (<cite>self.data</cite>).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.SequenceLabelingDataset.vectorization" title="graph4nlp.data.dataset.SequenceLabelingDataset.vectorization"><code class="xref py py-obj docutils literal notranslate"><span class="pre">vectorization</span></code></a>(data_items)</p></td>
<td><p>Convert tokens to indices which can be processed by downstream models.</p></td>
</tr>
</tbody>
</table>
<table class="docutils align-default">
<colgroup>
<col style="width: 77%" />
<col style="width: 23%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><strong>process_data_items</strong></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p><strong>register_datapipe_as_function</strong></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p><strong>register_function</strong></p></td>
<td></td>
</tr>
</tbody>
</table>
<dl class="method">
<dt id="graph4nlp.data.dataset.SequenceLabelingDataset.build_vocab">
<code class="sig-name descname">build_vocab</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.SequenceLabelingDataset.build_vocab" title="Permalink to this definition">¶</a></dt>
<dd><p>Build the vocabulary. If <cite>self.use_val_for_vocab</cite> is <cite>True</cite>,
use both training set and validation set for building
the vocabulary. Otherwise only the training set is used.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.SequenceLabelingDataset.collate_fn">
<em class="property">static </em><code class="sig-name descname">collate_fn</code><span class="sig-paren">(</span><em class="sig-param">data_list: [&lt;class 'graph4nlp.pytorch.data.dataset.SequenceLabelingDataItem'&gt;]</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.SequenceLabelingDataset.collate_fn" title="Permalink to this definition">¶</a></dt>
<dd><p>Takes a list of data and convert it to a batch of data.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.SequenceLabelingDataset.parse_file">
<code class="sig-name descname">parse_file</code><span class="sig-paren">(</span><em class="sig-param">file_path</em><span class="sig-paren">)</span> &#x2192; list<a class="headerlink" href="#graph4nlp.data.dataset.SequenceLabelingDataset.parse_file" title="Permalink to this definition">¶</a></dt>
<dd><p>Read and parse the file specified by <cite>file_path</cite>.
The file format is specified by each individual task-specific base class.
Returns all the indices of data items in this file w.r.t. the whole dataset.
For SequenceLabelingDataset, the format of the input file should contain lines of tokens,
each line representing one record of token at first column and its tag at the last column.
Read and parse the file specified by <cite>file_path</cite>. The file format is specified by each
individual task-specific base class. Returns all the indices of data items in
Read and parse the file specified by <cite>file_path</cite>. The file format is specified by each
individual task-specific base class. Returns all the indices of data items in
this file w.r.t. the whole dataset.
For SequenceLabelingDataset, the format of the input file should contain lines of tokens,
each line representing one record of token at first column and its tag at the last column.</p>
<p class="rubric">Examples</p>
<dl class="simple">
<dt>“EU       I-ORG</dt><dd><p>rejects  O
German   I-MISC”</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.SequenceLabelingDataset.vectorization">
<code class="sig-name descname">vectorization</code><span class="sig-paren">(</span><em class="sig-param">data_items</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.SequenceLabelingDataset.vectorization" title="Permalink to this definition">¶</a></dt>
<dd><p>Convert tokens to indices which can be processed by downstream models.</p>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="graph4nlp.data.dataset.KGCompletionDataset">
<em class="property">class </em><code class="sig-prename descclassname">graph4nlp.data.dataset.</code><code class="sig-name descname">KGCompletionDataset</code><span class="sig-paren">(</span><em class="sig-param">root_dir: str = None</em>, <em class="sig-param">topology_builder=None</em>, <em class="sig-param">topology_subdir: str = None</em>, <em class="sig-param">**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.KGCompletionDataset" title="Permalink to this definition">¶</a></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Attributes</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>processed_dir</strong></dt><dd></dd>
<dt><strong>processed_file_names</strong></dt><dd></dd>
<dt><strong>processed_file_paths</strong></dt><dd></dd>
<dt><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_dir</span></code></dt><dd><p>The directory where the raw data is stored.</p>
</dd>
<dt><strong>raw_file_names</strong></dt><dd></dd>
<dt><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_file_paths</span></code></dt><dd><p>The paths to raw files.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Methods</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.KGCompletionDataset.build_topology" title="graph4nlp.data.dataset.KGCompletionDataset.build_topology"><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_topology</span></code></a>(data_items)</p></td>
<td><p>Build graph topology for each item in the dataset.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.KGCompletionDataset.build_vocab" title="graph4nlp.data.dataset.KGCompletionDataset.build_vocab"><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_vocab</span></code></a>()</p></td>
<td><p>Build the vocabulary.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.KGCompletionDataset.collate_fn" title="graph4nlp.data.dataset.KGCompletionDataset.collate_fn"><code class="xref py py-obj docutils literal notranslate"><span class="pre">collate_fn</span></code></a>(data_list)</p></td>
<td><p>Takes a list of data and convert it to a batch of data.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">download</span></code>()</p></td>
<td><p>Download the raw data from the Internet.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.KGCompletionDataset.parse_file" title="graph4nlp.data.dataset.KGCompletionDataset.parse_file"><code class="xref py py-obj docutils literal notranslate"><span class="pre">parse_file</span></code></a>(file_path)</p></td>
<td><p>Read and parse the file specified by <cite>file_path</cite>.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.KGCompletionDataset.read_raw_data" title="graph4nlp.data.dataset.KGCompletionDataset.read_raw_data"><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_raw_data</span></code></a>()</p></td>
<td><p>Read raw data from the disk and put them in a dictionary (<cite>self.data</cite>).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.KGCompletionDataset.vectorization" title="graph4nlp.data.dataset.KGCompletionDataset.vectorization"><code class="xref py py-obj docutils literal notranslate"><span class="pre">vectorization</span></code></a>(data_items)</p></td>
<td><p>Convert tokens to indices which can be processed by downstream models.</p></td>
</tr>
</tbody>
</table>
<table class="docutils align-default">
<colgroup>
<col style="width: 77%" />
<col style="width: 23%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><strong>process_data_items</strong></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p><strong>register_datapipe_as_function</strong></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p><strong>register_function</strong></p></td>
<td></td>
</tr>
</tbody>
</table>
<dl class="method">
<dt id="graph4nlp.data.dataset.KGCompletionDataset.build_topology">
<code class="sig-name descname">build_topology</code><span class="sig-paren">(</span><em class="sig-param">data_items</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.KGCompletionDataset.build_topology" title="Permalink to this definition">¶</a></dt>
<dd><p>Build graph topology for each item in the dataset. The generated graph
is bound to the <cite>graph</cite> attribute of the DataItem.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.KGCompletionDataset.build_vocab">
<code class="sig-name descname">build_vocab</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.KGCompletionDataset.build_vocab" title="Permalink to this definition">¶</a></dt>
<dd><p>Build the vocabulary. If <cite>self.use_val_for_vocab</cite> is <cite>True</cite>,
use both training set and validation set for building
the vocabulary. Otherwise only the training set is used.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.KGCompletionDataset.collate_fn">
<em class="property">static </em><code class="sig-name descname">collate_fn</code><span class="sig-paren">(</span><em class="sig-param">data_list: [&lt;class 'graph4nlp.pytorch.data.dataset.KGCompletionDataItem'&gt;]</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.KGCompletionDataset.collate_fn" title="Permalink to this definition">¶</a></dt>
<dd><p>Takes a list of data and convert it to a batch of data.</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.KGCompletionDataset.parse_file">
<code class="sig-name descname">parse_file</code><span class="sig-paren">(</span><em class="sig-param">file_path</em><span class="sig-paren">)</span> &#x2192; list<a class="headerlink" href="#graph4nlp.data.dataset.KGCompletionDataset.parse_file" title="Permalink to this definition">¶</a></dt>
<dd><p>Read and parse the file specified by <cite>file_path</cite>. The file format
is specified by each individual task-specific base class. Returns
all the indices of data items in this file w.r.t. the whole dataset.</p>
<p>For KGCompletionDataset, the format of the input file should contain
lines of input, each line representing one record of data.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>file_path: str</strong></dt><dd><p>The path of the input file.</p>
</dd>
</dl>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><dl class="simple">
<dt>list</dt><dd><p>The indices of data items in the file w.r.t. the whole dataset.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Examples</p>
<p>input:
{“e1”: “person100”, “e2”: “None”, “rel”: “term6”, “rel_eval”: “None”,
“e2_multi1”: “person90 person80 person59 person82 person63 person77
person85 person83 person56”, “e2_multi2”: “None”}</p>
<dl class="simple">
<dt>DataItem: e1=”person100”</dt><dd><p>e2=”None”
rel=”term6”
…</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.KGCompletionDataset.read_raw_data">
<code class="sig-name descname">read_raw_data</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.KGCompletionDataset.read_raw_data" title="Permalink to this definition">¶</a></dt>
<dd><p>Read raw data from the disk and put them in a dictionary (<cite>self.data</cite>).
The raw data file should be organized as the format defined in <cite>self.parse_file()</cite> method.</p>
<p>This function calls <cite>self.parse_file()</cite> repeatedly and pass the file paths in
<cite>self.raw_file_names</cite> once at a time.</p>
<p>This function builds <cite>self.data</cite> which is a dict of {int (index): DataItem},
where the id represents the index of the DataItem w.r.t. the whole dataset.</p>
<p>This function also builds the <cite>self.split_ids</cite> dictionary whose keys correspond to
those of self.raw_file_names defined by the user, indicating the indices of each
subset (e.g. train, val and test).</p>
</dd></dl>

<dl class="method">
<dt id="graph4nlp.data.dataset.KGCompletionDataset.vectorization">
<code class="sig-name descname">vectorization</code><span class="sig-paren">(</span><em class="sig-param">data_items</em><span class="sig-paren">)</span><a class="headerlink" href="#graph4nlp.data.dataset.KGCompletionDataset.vectorization" title="Permalink to this definition">¶</a></dt>
<dd><p>Convert tokens to indices which can be processed by downstream models.</p>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="base-dataset-class">
<h2>Base Dataset Class<a class="headerlink" href="#base-dataset-class" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt>
<em class="property">class </em><code class="sig-prename descclassname">graph4nlp.data.dataset.</code><code class="sig-name descname">Dataset</code><span class="sig-paren">(</span><em class="sig-param">root</em>, <em class="sig-param">topology_builder</em>, <em class="sig-param">topology_subdir</em>, <em class="sig-param">tokenizer=&lt;function word_tokenize&gt;</em>, <em class="sig-param">lower_case=True</em>, <em class="sig-param">pretrained_word_emb_name='840B'</em>, <em class="sig-param">pretrained_word_emb_url=None</em>, <em class="sig-param">target_pretrained_word_emb_name=None</em>, <em class="sig-param">target_pretrained_word_emb_url=None</em>, <em class="sig-param">pretrained_word_emb_cache_dir='.vector_cache/'</em>, <em class="sig-param">max_word_vocab_size=None</em>, <em class="sig-param">min_word_vocab_freq=1</em>, <em class="sig-param">use_val_for_vocab=False</em>, <em class="sig-param">seed=1234</em>, <em class="sig-param">thread_number=4</em>, <em class="sig-param">port=9000</em>, <em class="sig-param">timeout=15000</em>, <em class="sig-param">for_inference=False</em>, <em class="sig-param">reused_vocab_model=None</em>, <em class="sig-param">**kwargs</em><span class="sig-paren">)</span></dt>
<dd><p>Base class for datasets.</p>
<p>The dataset is organized in a two-layer index style. Direct access
to the dataset object, e.g. Dataset[1], will first be converted to
the access to the internal index list, which is then passed to
access the actual data. This design is for the ease of sampling.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>root: str</strong></dt><dd><p>The root directory path where the dataset is stored.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Examples</p>
<p>Suppose we have a Dataset containing 5 data items
[‘a’, ‘b’, ‘c’, ‘d’, ‘e’]. The indices of the 5 elements in the
list are correspondingly [0, 1, 2, 3, 4]. Suppose the dataset
is shuffled, which shuffles the internal index list, the consequent
indices becomes [2, 3, 1, 4, 5]. Then an access to the dataset
<cite>Dataset[2]</cite> will first access the indices[2] which is 1, and then
use the received index to access the actual dataset, which will
return the actual data item ‘b’. Now to the user the 3rd ([2])
element in the dataset got shuffled and is not ‘c’.</p>
<dl class="field-list simple">
<dt class="field-odd">Attributes</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>processed_dir</strong></dt><dd></dd>
<dt><strong>processed_file_names</strong></dt><dd></dd>
<dt><strong>processed_file_paths</strong></dt><dd></dd>
<dt><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.raw_dir" title="graph4nlp.data.dataset.Dataset.raw_dir"><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_dir</span></code></a></dt><dd><p>The directory where the raw data is stored.</p>
</dd>
<dt><strong>raw_file_names</strong></dt><dd></dd>
<dt><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.raw_file_paths" title="graph4nlp.data.dataset.Dataset.raw_file_paths"><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_file_paths</span></code></a></dt><dd><p>The paths to raw files.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Methods</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.build_topology" title="graph4nlp.data.dataset.Dataset.build_topology"><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_topology</span></code></a>(data_items)</p></td>
<td><p>Build graph topology for each item in the dataset.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.build_vocab" title="graph4nlp.data.dataset.Dataset.build_vocab"><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_vocab</span></code></a>()</p></td>
<td><p>Build the vocabulary.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.collate_fn" title="graph4nlp.data.dataset.Dataset.collate_fn"><code class="xref py py-obj docutils literal notranslate"><span class="pre">collate_fn</span></code></a>(data_list)</p></td>
<td><p>Takes a list of data and convert it to a batch of data.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.download" title="graph4nlp.data.dataset.Dataset.download"><code class="xref py py-obj docutils literal notranslate"><span class="pre">download</span></code></a>()</p></td>
<td><p>Download the raw data from the Internet.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.parse_file" title="graph4nlp.data.dataset.Dataset.parse_file"><code class="xref py py-obj docutils literal notranslate"><span class="pre">parse_file</span></code></a>(file_path)</p></td>
<td><p>To be implemented in task-specific dataset base class.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.read_raw_data" title="graph4nlp.data.dataset.Dataset.read_raw_data"><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_raw_data</span></code></a>()</p></td>
<td><p>Read raw data from the disk and put them in a dictionary (<cite>self.data</cite>).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Dataset.vectorization" title="graph4nlp.data.dataset.Dataset.vectorization"><code class="xref py py-obj docutils literal notranslate"><span class="pre">vectorization</span></code></a>(data_items)</p></td>
<td><p>Convert tokens to indices which can be processed by downstream models.</p></td>
</tr>
</tbody>
</table>
<table class="docutils align-default">
<colgroup>
<col style="width: 77%" />
<col style="width: 23%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><strong>process_data_items</strong></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p><strong>register_datapipe_as_function</strong></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p><strong>register_function</strong></p></td>
<td></td>
</tr>
</tbody>
</table>
<dl class="method">
<dt>
<code class="sig-name descname">build_topology</code><span class="sig-paren">(</span><em class="sig-param">data_items</em><span class="sig-paren">)</span></dt>
<dd><p>Build graph topology for each item in the dataset. The generated graph
is bound to the <cite>graph</cite> attribute of the DataItem.</p>
</dd></dl>

<dl class="method">
<dt>
<code class="sig-name descname">build_vocab</code><span class="sig-paren">(</span><span class="sig-paren">)</span></dt>
<dd><p>Build the vocabulary. If <cite>self.use_val_for_vocab</cite> is <cite>True</cite>,
use both training set and validation set for building
the vocabulary. Otherwise only the training set is used.</p>
</dd></dl>

<dl class="method">
<dt>
<em class="property">abstract static </em><code class="sig-name descname">collate_fn</code><span class="sig-paren">(</span><em class="sig-param">data_list</em><span class="sig-paren">)</span></dt>
<dd><p>Takes a list of data and convert it to a batch of data.</p>
</dd></dl>

<dl class="method">
<dt>
<em class="property">abstract </em><code class="sig-name descname">download</code><span class="sig-paren">(</span><span class="sig-paren">)</span></dt>
<dd><p>Download the raw data from the Internet.</p>
</dd></dl>

<dl class="method">
<dt>
<em class="property">abstract </em><code class="sig-name descname">parse_file</code><span class="sig-paren">(</span><em class="sig-param">file_path</em><span class="sig-paren">)</span></dt>
<dd><p>To be implemented in task-specific dataset base class.</p>
</dd></dl>

<dl class="method">
<dt>
<em class="property">property </em><code class="sig-name descname">raw_dir</code></dt>
<dd><p>The directory where the raw data is stored.</p>
</dd></dl>

<dl class="method">
<dt>
<em class="property">property </em><code class="sig-name descname">raw_file_paths</code></dt>
<dd><p>The paths to raw files.</p>
</dd></dl>

<dl class="method">
<dt>
<code class="sig-name descname">read_raw_data</code><span class="sig-paren">(</span><span class="sig-paren">)</span></dt>
<dd><p>Read raw data from the disk and put them in a dictionary (<cite>self.data</cite>).
The raw data file should be organized as the format defined in <cite>self.parse_file()</cite> method.</p>
<p>This function calls <cite>self.parse_file()</cite> repeatedly and pass the file paths in
<cite>self.raw_file_names</cite> once at a time.</p>
<p>This function builds <cite>self.data</cite> which is a dict of {int (index): DataItem},
where the id represents the index of the DataItem w.r.t. the whole dataset.</p>
<p>This function also builds the <cite>self.split_ids</cite> dictionary whose keys correspond to
those of self.raw_file_names defined by the user, indicating the indices of each
subset (e.g. train, val and test).</p>
</dd></dl>

<dl class="method">
<dt>
<em class="property">abstract </em><code class="sig-name descname">vectorization</code><span class="sig-paren">(</span><em class="sig-param">data_items</em><span class="sig-paren">)</span></dt>
<dd><p>Convert tokens to indices which can be processed by downstream models.</p>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="task-level-dataset-base-class">
<h2>Task Level Dataset Base Class<a class="headerlink" href="#task-level-dataset-base-class" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt>
<em class="property">class </em><code class="sig-prename descclassname">graph4nlp.data.dataset.</code><code class="sig-name descname">Text2LabelDataset</code><span class="sig-paren">(</span><em class="sig-param">graph_name: str</em>, <em class="sig-param">root_dir: str = None</em>, <em class="sig-param">static_or_dynamic: str = None</em>, <em class="sig-param">topology_builder: Union[graph4nlp.pytorch.modules.graph_construction.base.StaticGraphConstructionBase</em>, <em class="sig-param">graph4nlp.pytorch.modules.graph_construction.base.DynamicGraphConstructionBase] = &lt;class 'graph4nlp.pytorch.modules.graph_construction.dependency_graph_construction.DependencyBasedGraphConstruction'&gt;</em>, <em class="sig-param">topology_subdir: str = None</em>, <em class="sig-param">dynamic_init_graph_name: str = None</em>, <em class="sig-param">dynamic_init_topology_builder: graph4nlp.pytorch.modules.graph_construction.base.StaticGraphConstructionBase = None</em>, <em class="sig-param">dynamic_init_topology_aux_args=None</em>, <em class="sig-param">**kwargs</em><span class="sig-paren">)</span></dt>
<dd><p>The dataset for text-to-label applications.
:param graph_name: The name of graph construction method. E.g., “dependency”.</p>
<blockquote>
<div><dl class="simple">
<dt>Note that if it is in the provided graph names (i.e., “dependency”,             “constituency”, “ie”, “node_emb”, “node_emb_refine”), the following             parameters are set by default and users can’t modify them:</dt><dd><ol class="arabic simple">
<li><p><code class="docutils literal notranslate"><span class="pre">topology_builder</span></code></p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">static_or_dynamic</span></code></p></li>
</ol>
</dd>
</dl>
<p>If you need to customize your graph construction method, you should rename the             <code class="docutils literal notranslate"><span class="pre">graph_name</span></code> and set the parameters above.</p>
</div></blockquote>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>root_dir</strong> (<em>str</em><em>, </em><em>default=None</em>) – The path of dataset.</p></li>
<li><p><strong>topology_builder</strong> (<em>Union</em><em>[</em><em>StaticGraphConstructionBase</em><em>, </em><em>DynamicGraphConstructionBase</em><em>]</em><em>, </em><em>default=None</em>) – The graph construction class.</p></li>
<li><p><strong>topology_subdir</strong> (<em>str</em>) – The directory name of processed path.</p></li>
<li><p><strong>static_or_dynamic</strong> (<em>str</em><em>, </em><em>default='static'</em>) – The graph type. Expected in (‘static’, ‘dynamic’)</p></li>
<li><p><strong>dynamic_init_graph_name</strong> (<em>str</em><em>, </em><em>default=None</em>) – <p>The graph name of the initial graph. Expected in (None, “line”,             “dependency”, “constituency”).
Note that if it is in the provided graph names (i.e., “line”, “dependency”,             “constituency”), the following parameters are set by default and users             can’t modify them:</p>
<blockquote>
<div><ol class="arabic simple">
<li><p><code class="docutils literal notranslate"><span class="pre">dynamic_init_topology_builder</span></code></p></li>
</ol>
</div></blockquote>
<p>If you need to customize your graph construction method, you should rename the             <code class="docutils literal notranslate"><span class="pre">graph_name</span></code> and set the parameters above.</p>
</p></li>
<li><p><strong>dynamic_init_topology_builder</strong> (<em>StaticGraphConstructionBase</em>) – The graph construction class.</p></li>
<li><p><strong>dynamic_init_topology_aux_args</strong> (<em>None</em><em>,</em>) – TBD.</p></li>
</ul>
</dd>
<dt class="field-even">Attributes</dt>
<dd class="field-even"><dl class="simple">
<dt><strong>processed_dir</strong></dt><dd></dd>
<dt><strong>processed_file_names</strong></dt><dd></dd>
<dt><strong>processed_file_paths</strong></dt><dd></dd>
<dt><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_dir</span></code></dt><dd><p>The directory where the raw data is stored.</p>
</dd>
<dt><strong>raw_file_names</strong></dt><dd></dd>
<dt><code class="xref py py-obj docutils literal notranslate"><span class="pre">raw_file_paths</span></code></dt><dd><p>The paths to raw files.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Methods</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_topology</span></code>(data_items)</p></td>
<td><p>Build graph topology for each item in the dataset.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Text2LabelDataset.build_vocab" title="graph4nlp.data.dataset.Text2LabelDataset.build_vocab"><code class="xref py py-obj docutils literal notranslate"><span class="pre">build_vocab</span></code></a>()</p></td>
<td><p>Build the vocabulary.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Text2LabelDataset.collate_fn" title="graph4nlp.data.dataset.Text2LabelDataset.collate_fn"><code class="xref py py-obj docutils literal notranslate"><span class="pre">collate_fn</span></code></a>(data_list)</p></td>
<td><p>Takes a list of data and convert it to a batch of data.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">download</span></code>()</p></td>
<td><p>Download the raw data from the Internet.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Text2LabelDataset.parse_file" title="graph4nlp.data.dataset.Text2LabelDataset.parse_file"><code class="xref py py-obj docutils literal notranslate"><span class="pre">parse_file</span></code></a>(file_path)</p></td>
<td><p>Read and parse the file specified by <cite>file_path</cite>.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">read_raw_data</span></code>()</p></td>
<td><p>Read raw data from the disk and put them in a dictionary (<cite>self.data</cite>).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="#graph4nlp.data.dataset.Text2LabelDataset.vectorization" title="graph4nlp.data.dataset.Text2LabelDataset.vectorization"><code class="xref py py-obj docutils literal notranslate"><span class="pre">vectorization</span></code></a>(data_items)</p></td>
<td><p>Convert tokens to indices which can be processed by downstream models.</p></td>
</tr>
</tbody>
</table>
<table class="docutils align-default">
<colgroup>
<col style="width: 77%" />
<col style="width: 23%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><strong>process_data_items</strong></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p><strong>register_datapipe_as_function</strong></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p><strong>register_function</strong></p></td>
<td></td>
</tr>
</tbody>
</table>
<dl class="method">
<dt>
<code class="sig-name descname">build_vocab</code><span class="sig-paren">(</span><span class="sig-paren">)</span></dt>
<dd><p>Build the vocabulary. If <cite>self.use_val_for_vocab</cite> is <cite>True</cite>,
use both training set and validation set for building
the vocabulary. Otherwise only the training set is used.</p>
</dd></dl>

<dl class="method">
<dt>
<em class="property">static </em><code class="sig-name descname">collate_fn</code><span class="sig-paren">(</span><em class="sig-param">data_list: [&lt;class 'graph4nlp.pytorch.data.dataset.Text2LabelDataItem'&gt;]</em><span class="sig-paren">)</span></dt>
<dd><p>Takes a list of data and convert it to a batch of data.</p>
</dd></dl>

<dl class="method">
<dt>
<code class="sig-name descname">parse_file</code><span class="sig-paren">(</span><em class="sig-param">file_path</em><span class="sig-paren">)</span> &#x2192; list</dt>
<dd><p>Read and parse the file specified by <cite>file_path</cite>. The file format
is specified by each individual task-specific base class. Returns
all the indices of data items in this file w.r.t. the whole dataset.</p>
<p>For Text2LabelDataset, the format of the input file should contain
lines of input, each line representing one record of data. The
input and output is separated by a tab( ).</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><dl class="simple">
<dt><strong>file_path: str</strong></dt><dd><p>The path of the input file.</p>
</dd>
</dl>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><dl class="simple">
<dt>list</dt><dd><p>The indices of data items in the file w.r.t. the whole dataset.</p>
</dd>
</dl>
</dd>
</dl>
<p class="rubric">Examples</p>
<p>input: How far is it from Denver to Aspen ?    NUM</p>
<p>DataItem: input_text=”How far is it from Denver to Aspen ?”, output_label=”NUM”</p>
</dd></dl>

<dl class="method">
<dt>
<code class="sig-name descname">vectorization</code><span class="sig-paren">(</span><em class="sig-param">data_items</em><span class="sig-paren">)</span></dt>
<dd><p>Convert tokens to indices which can be processed by downstream models.</p>
</dd></dl>

</dd></dl>

</div>
</div>


           </div>
          </div>
          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
        <a href="../guide/evaluation.html" class="btn btn-neutral float-left" title="Chapter 7. Evaluations and Loss components" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
        <a href="datasets.html" class="btn btn-neutral float-right" title="graph4nlp.datasets" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
    </div>

  <hr/>

  <div role="contentinfo">
    <p>&#169; Copyright 2020, Graph4AI Group.</p>
  </div>

  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
    provided by <a href="https://readthedocs.org">Read the Docs</a>.
   

</footer>
        </div>
      </div>
    </section>
  </div>
  <script>
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script> 

</body>
</html>